{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"mc_dropout_heteroscedastic.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"id":"xAYPKSFrG8AF","colab_type":"code","outputId":"cf3df00b-2fcc-48ba-bd75-af624f50aa20","executionInfo":{"status":"ok","timestamp":1558128143256,"user_tz":-60,"elapsed":10811,"user":{"displayName":"Stratis Markou","photoUrl":"","userId":"09754366312766083286"}},"colab":{"base_uri":"https://localhost:8080/","height":258}},"source":["!pip3 install http://download.pytorch.org/whl/cu92/torch-0.4.1-cp36-cp36m-linux_x86_64.whl\n","!pip3 install torchvision\n","!pip3 install GPy\n","\n","import pandas as pd\n","import zipfile\n","import urllib.request\n","import os\n","import GPy\n","import time\n","import copy\n","import math\n","import matplotlib.pyplot as plt\n","import numpy as np\n","import seaborn as sns\n","import torch\n","import torch.nn as nn\n","import torch.nn.functional as F\n","from torch.autograd import Variable\n","from torch.optim import Optimizer\n","from torch.optim.sgd import SGD\n","from sklearn.model_selection import KFold\n","\n","from torchvision import datasets, transforms\n","from torchvision.utils import make_grid\n","from tqdm import tqdm, trange\n","from google.colab import files\n","%config InlineBackend.figure_format = 'svg'"],"execution_count":25,"outputs":[{"output_type":"stream","text":["Requirement already satisfied: torch==0.4.1 from http://download.pytorch.org/whl/cu92/torch-0.4.1-cp36-cp36m-linux_x86_64.whl in /usr/local/lib/python3.6/dist-packages (0.4.1)\n","Requirement already satisfied: torchvision in /usr/local/lib/python3.6/dist-packages (0.2.2.post3)\n","Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from torchvision) (1.12.0)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torchvision) (1.16.3)\n","Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from torchvision) (4.3.0)\n","Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (from torchvision) (0.4.1)\n","Requirement already satisfied: olefile in /usr/local/lib/python3.6/dist-packages (from pillow>=4.1.1->torchvision) (0.46)\n","Requirement already satisfied: GPy in /usr/local/lib/python3.6/dist-packages (1.9.6)\n","Requirement already satisfied: numpy>=1.7 in /usr/local/lib/python3.6/dist-packages (from GPy) (1.16.3)\n","Requirement already satisfied: paramz>=0.9.0 in /usr/local/lib/python3.6/dist-packages (from GPy) (0.9.5)\n","Requirement already satisfied: scipy>=0.16 in /usr/local/lib/python3.6/dist-packages (from GPy) (1.2.1)\n","Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from GPy) (1.12.0)\n","Requirement already satisfied: decorator>=4.0.10 in /usr/local/lib/python3.6/dist-packages (from paramz>=0.9.0->GPy) (4.4.0)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"1-7dNcVmHA3I","colab_type":"code","outputId":"77b79c8c-07bb-4329-f5f1-24d350d958f6","executionInfo":{"status":"ok","timestamp":1558128143257,"user_tz":-60,"elapsed":10522,"user":{"displayName":"Stratis Markou","photoUrl":"","userId":"09754366312766083286"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["torch.cuda.device(0)\n","torch.cuda.get_device_name(torch.cuda.current_device())"],"execution_count":26,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'Tesla T4'"]},"metadata":{"tags":[]},"execution_count":26}]},{"cell_type":"code","metadata":{"id":"_1S5kt0omQ-N","colab_type":"code","colab":{}},"source":["def to_variable(var=(), cuda=True, volatile=False):\n","    out = []\n","    for v in var:\n","        \n","        if isinstance(v, np.ndarray):\n","            v = torch.from_numpy(v).type(torch.FloatTensor)\n","\n","        if not v.is_cuda and cuda:\n","            v = v.cuda()\n","\n","        if not isinstance(v, Variable):\n","            v = Variable(v, volatile=volatile)\n","\n","        out.append(v)\n","    return out"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Va8V78eFFsc9","colab_type":"code","colab":{}},"source":["def log_gaussian_loss(output, target, sigma, no_dim):\n","    exponent = -0.5*(target - output)**2/sigma**2\n","    log_coeff = -no_dim*torch.log(sigma) - 0.5*no_dim*np.log(2*np.pi)\n","    \n","    return - (log_coeff + exponent).sum()\n","\n","\n","def get_kl_divergence(weights, prior, varpost):\n","    prior_loglik = prior.loglik(weights)\n","    \n","    varpost_loglik = varpost.loglik(weights)\n","    varpost_lik = varpost_loglik.exp()\n","    \n","    return (varpost_lik*(varpost_loglik - prior_loglik)).sum()\n","\n","\n","class gaussian:\n","    def __init__(self, mu, sigma):\n","        self.mu = mu\n","        self.sigma = sigma\n","        \n","    def loglik(self, weights):\n","        exponent = -0.5*(weights - self.mu)**2/self.sigma**2\n","        log_coeff = -0.5*(np.log(2*np.pi) + 2*np.log(self.sigma))\n","        \n","        return (exponent + log_coeff).sum()\n","        "],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"colab_type":"code","id":"_8dV-QIq5G-I","colab":{}},"source":["class MC_Dropout_Model(nn.Module):\n","    def __init__(self, input_dim, output_dim, num_units, drop_prob):\n","        super(MC_Dropout_Model, self).__init__()\n","        \n","        self.input_dim = input_dim\n","        self.output_dim = output_dim\n","        self.drop_prob = drop_prob\n","        \n","        # network with two hidden and one output layer\n","        self.layer1 = nn.Linear(input_dim, num_units)\n","        self.layer2 = nn.Linear(num_units, 2*output_dim)\n","        \n","        self.activation = nn.ReLU(inplace = True)\n","\n","    \n","    def forward(self, x):\n","        \n","        x = x.view(-1, self.input_dim)\n","        \n","        x = self.layer1(x)\n","        x = self.activation(x)\n","        \n","        x = F.dropout(x, p=self.drop_prob, training=True)\n","        \n","        x = self.layer2(x)\n","        \n","        return x"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"colab_type":"code","id":"oAYelw3B5G-K","colab":{}},"source":["class MC_Dropout_Wrapper:\n","    def __init__(self, network, learn_rate, batch_size, weight_decay):\n","        \n","        self.learn_rate = learn_rate\n","        self.batch_size = batch_size\n","        \n","        self.network = network\n","        self.network.cuda()\n","        \n","        self.optimizer = torch.optim.SGD(self.network.parameters(), lr=learn_rate, weight_decay=weight_decay)\n","        self.loss_func = log_gaussian_loss\n","    \n","    def fit(self, x, y):\n","        x, y = to_variable(var=(x, y), cuda=True)\n","        \n","        # reset gradient and total loss\n","        self.optimizer.zero_grad()\n","        \n","        output = self.network(x)\n","        loss = self.loss_func(output[:, :1], y, output[:, 1:].exp(), 1)\n","        \n","        loss.backward()\n","        self.optimizer.step()\n","\n","        return loss\n","    \n","    def get_loss_and_rmse(self, x, y, num_samples):\n","        x, y = to_variable(var=(x, y), cuda=True)\n","        \n","        means, stds = [], []\n","        for i in range(num_samples):\n","            output = self.network(x)\n","            means.append(output[:, :1])\n","            stds.append(output[:, 1:].exp())\n","        \n","        means, stds = torch.cat(means, dim=1), torch.cat(stds, dim=1)\n","        mean = means.mean(dim=-1)[:, None]\n","        std = ((means.var(dim=-1) + stds.mean(dim=-1)**2)**0.5)[:, None]\n","        loss = self.loss_func(mean, y, std, 1)\n","        \n","        rmse = ((mean - y)**2).mean()**0.5\n","\n","        return loss.detach().cpu(), rmse.detach().cpu()"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"ym6HBK-s8GnO","colab_type":"code","outputId":"9feb3d2f-f4ff-4ed4-ee33-5bf605f5a282","executionInfo":{"status":"ok","timestamp":1558130333486,"user_tz":-60,"elapsed":3887,"user":{"displayName":"Stratis Markou","photoUrl":"","userId":"09754366312766083286"}},"colab":{"base_uri":"https://localhost:8080/","height":204}},"source":["np.random.seed(2)\n","no_points = 400\n","lengthscale = 1\n","variance = 1.0\n","sig_noise = 0.3\n","x = np.random.uniform(-3, 3, no_points)[:, None]\n","x.sort(axis = 0)\n","\n","\n","k = GPy.kern.RBF(input_dim=1, variance=variance, lengthscale=lengthscale)\n","C = k.K(x, x) + np.eye(no_points)*(x + 2)**2*sig_noise**2\n","\n","y = np.random.multivariate_normal(np.zeros((no_points)), C)[:, None]\n","y = (y - y.mean())\n","x_train = x[75:325]\n","y_train = y[75:325]\n","\n","print(x_train.shape, y_train.shape)\n","num_epochs, batch_size = 2000, len(x_train)\n","\n","net = MC_Dropout_Wrapper(network=MC_Dropout_Model(input_dim=1, output_dim=1, num_units=200, drop_prob=0.5),\n","                         learn_rate=1e-4, batch_size=batch_size, weight_decay=1e-2)\n","\n","fit_loss_train = np.zeros(num_epochs)\n","best_net, best_loss = None, float('inf')\n","nets, losses = [], []\n","\n","for i in range(num_epochs):\n","    \n","    loss = net.fit(x_train, y_train)\n","    \n","    if i % 200 == 0:\n","        print('Epoch: %4d, Train loss = %7.3f' % (i, loss.cpu().data.numpy()/batch_size))"],"execution_count":167,"outputs":[{"output_type":"stream","text":["(250, 1) (250, 1)\n","Epoch:    0, Train loss =   2.718\n","Epoch:  200, Train loss =   1.177\n","Epoch:  400, Train loss =   1.448\n","Epoch:  600, Train loss =   1.698\n","Epoch:  800, Train loss =   1.527\n","Epoch: 1000, Train loss =   1.614\n","Epoch: 1200, Train loss =   1.085\n","Epoch: 1400, Train loss =   1.118\n","Epoch: 1600, Train loss =   0.952\n","Epoch: 1800, Train loss =   0.893\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"tvP-HbNH_Uen","colab_type":"code","outputId":"8b281638-0998-4d35-f260-cb97ad83d73b","executionInfo":{"status":"ok","timestamp":1558130335941,"user_tz":-60,"elapsed":6190,"user":{"displayName":"Stratis Markou","photoUrl":"","userId":"09754366312766083286"}},"colab":{"base_uri":"https://localhost:8080/","height":550}},"source":["samples = []\n","noises = []\n","for i in range(1000):\n","    preds = net.network.forward(torch.linspace(-5, 5, 200).cuda()).cpu().data.numpy()\n","    samples.append(preds[:, 0])\n","    noises.append(np.exp(preds[:, 1]))\n","    \n","samples = np.array(samples)\n","noises = np.array(noises)\n","means = (samples.mean(axis = 0)).reshape(-1)\n","\n","aleatoric = (noises**2).mean(axis = 0)**0.5\n","epistemic = (samples.var(axis = 0)**0.5).reshape(-1)\n","total_unc = (aleatoric**2 + epistemic**2)**0.5\n","\n","\n","c = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd',\n","     '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf']\n","\n","plt.figure(figsize = (6, 5))\n","plt.style.use('default')\n","plt.scatter(x_train, y_train, s = 10, marker = 'x', color = 'black', alpha = 0.5)\n","plt.fill_between(np.linspace(-5, 5, 200), means + aleatoric, means + total_unc, color = c[0], alpha = 0.3, label = 'Epistemic + Aleatoric')\n","plt.fill_between(np.linspace(-5, 5, 200), means - total_unc, means - aleatoric, color = c[0], alpha = 0.3)\n","plt.fill_between(np.linspace(-5, 5, 200), means - aleatoric, means + aleatoric, color = c[1], alpha = 0.4, label = 'Aleatoric')\n","plt.plot(np.linspace(-5, 5, 200), means, color = 'black', linewidth = 1)\n","plt.xlim([-5, 5])\n","plt.ylim([-5, 7])\n","plt.xlabel('$x$', fontsize=30)\n","plt.title('MC dropout', fontsize=40)\n","plt.tick_params(labelsize=30)\n","plt.xticks(np.arange(-4, 5, 2))\n","plt.yticks(np.arange(-4, 7, 2))\n","plt.gca().set_yticklabels([])\n","plt.gca().yaxis.grid(alpha=0.3)\n","plt.gca().xaxis.grid(alpha=0.3)\n","plt.savefig('mc_dropout_hetero.pdf', bbox_inches = 'tight')\n","\n","files.download(\"mc_dropout_hetero.pdf\")\n","\n","plt.show()"],"execution_count":168,"outputs":[{"output_type":"display_data","data":{"text/plain":["<Figure size 600x500 with 1 Axes>"],"image/svg+xml":"<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Created with matplotlib (https://matplotlib.org/) -->\n<svg height=\"397.0625pt\" version=\"1.1\" viewBox=\"0 0 356.2 397.0625\" width=\"356.2pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n <defs>\n  <style type=\"text/css\">\n*{stroke-linecap:butt;stroke-linejoin:round;}\n  </style>\n </defs>\n <g id=\"figure_1\">\n  <g id=\"patch_1\">\n   <path d=\"M 0 397.0625 \nL 356.2 397.0625 \nL 356.2 0 \nL 0 0 \nz\n\" style=\"fill:#ffffff;\"/>\n  </g>\n  <g id=\"axes_1\">\n   <g id=\"patch_2\">\n    <path d=\"M 10.7 320.79375 \nL 345.5 320.79375 \nL 345.5 43.59375 \nL 10.7 43.59375 \nz\n\" style=\"fill:#ffffff;\"/>\n   </g>\n   <g id=\"PathCollection_1\">\n    <defs>\n     <path d=\"M -1.581139 1.581139 \nL 1.581139 -1.581139 \nM -1.581139 -1.581139 \nL 1.581139 1.581139 \n\" id=\"mb4c86e9e83\" style=\"stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\"/>\n    </defs>\n    <g clip-path=\"url(#pebacec704d)\">\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"115.282121\" xlink:href=\"#mb4c86e9e83\" y=\"186.530464\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"116.790285\" xlink:href=\"#mb4c86e9e83\" y=\"183.887151\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"117.226193\" xlink:href=\"#mb4c86e9e83\" y=\"187.105861\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"117.439686\" xlink:href=\"#mb4c86e9e83\" y=\"183.718285\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"117.61485\" xlink:href=\"#mb4c86e9e83\" y=\"186.349227\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"118.00521\" xlink:href=\"#mb4c86e9e83\" y=\"185.73061\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"118.186179\" xlink:href=\"#mb4c86e9e83\" y=\"183.947491\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"118.368141\" xlink:href=\"#mb4c86e9e83\" y=\"184.867541\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"118.769818\" xlink:href=\"#mb4c86e9e83\" y=\"184.527663\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"118.999834\" xlink:href=\"#mb4c86e9e83\" y=\"178.796332\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"119.060893\" xlink:href=\"#mb4c86e9e83\" y=\"183.212777\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"119.252172\" xlink:href=\"#mb4c86e9e83\" y=\"184.219778\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"119.543981\" xlink:href=\"#mb4c86e9e83\" y=\"184.495977\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"120.165723\" xlink:href=\"#mb4c86e9e83\" y=\"178.725971\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"120.776029\" xlink:href=\"#mb4c86e9e83\" y=\"183.573258\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"120.83401\" xlink:href=\"#mb4c86e9e83\" y=\"181.683752\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"121.331799\" xlink:href=\"#mb4c86e9e83\" y=\"179.370285\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"121.61672\" xlink:href=\"#mb4c86e9e83\" y=\"180.211518\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"121.911832\" xlink:href=\"#mb4c86e9e83\" y=\"180.75437\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"121.915111\" xlink:href=\"#mb4c86e9e83\" y=\"182.381654\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"122.02192\" xlink:href=\"#mb4c86e9e83\" y=\"182.40679\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"122.03062\" xlink:href=\"#mb4c86e9e83\" y=\"178.407944\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"123.061291\" xlink:href=\"#mb4c86e9e83\" y=\"176.78924\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"123.583227\" xlink:href=\"#mb4c86e9e83\" y=\"179.086346\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"123.776831\" xlink:href=\"#mb4c86e9e83\" y=\"180.272072\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"124.287914\" xlink:href=\"#mb4c86e9e83\" y=\"177.791056\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"126.874423\" xlink:href=\"#mb4c86e9e83\" y=\"174.116515\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"127.038547\" xlink:href=\"#mb4c86e9e83\" y=\"173.738497\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"127.127421\" xlink:href=\"#mb4c86e9e83\" y=\"172.57505\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"127.312964\" xlink:href=\"#mb4c86e9e83\" y=\"179.944508\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"127.38802\" xlink:href=\"#mb4c86e9e83\" y=\"171.341147\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"127.625744\" xlink:href=\"#mb4c86e9e83\" y=\"169.184865\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"128.137713\" xlink:href=\"#mb4c86e9e83\" y=\"177.296353\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"128.592776\" xlink:href=\"#mb4c86e9e83\" y=\"176.043459\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"128.729058\" xlink:href=\"#mb4c86e9e83\" y=\"178.345503\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"129.784424\" xlink:href=\"#mb4c86e9e83\" y=\"170.068533\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"129.837526\" xlink:href=\"#mb4c86e9e83\" y=\"176.730886\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"130.033438\" xlink:href=\"#mb4c86e9e83\" y=\"173.804983\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"130.647831\" xlink:href=\"#mb4c86e9e83\" y=\"168.952258\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"130.982285\" xlink:href=\"#mb4c86e9e83\" y=\"168.055986\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"131.219809\" xlink:href=\"#mb4c86e9e83\" y=\"171.944648\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"131.260263\" xlink:href=\"#mb4c86e9e83\" y=\"172.595029\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"131.774822\" xlink:href=\"#mb4c86e9e83\" y=\"167.831127\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"132.713023\" xlink:href=\"#mb4c86e9e83\" y=\"170.575433\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"133.960294\" xlink:href=\"#mb4c86e9e83\" y=\"172.542789\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"134.561016\" xlink:href=\"#mb4c86e9e83\" y=\"174.749289\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"134.660867\" xlink:href=\"#mb4c86e9e83\" y=\"168.830532\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"135.032078\" xlink:href=\"#mb4c86e9e83\" y=\"162.210277\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"135.440171\" xlink:href=\"#mb4c86e9e83\" y=\"169.783722\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"135.850013\" xlink:href=\"#mb4c86e9e83\" y=\"163.734123\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"136.000924\" xlink:href=\"#mb4c86e9e83\" y=\"170.867358\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"136.183921\" xlink:href=\"#mb4c86e9e83\" y=\"170.283069\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"136.522507\" xlink:href=\"#mb4c86e9e83\" y=\"169.364247\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"136.954213\" xlink:href=\"#mb4c86e9e83\" y=\"167.589275\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"137.223481\" xlink:href=\"#mb4c86e9e83\" y=\"164.714394\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"137.854631\" xlink:href=\"#mb4c86e9e83\" y=\"178.263436\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"138.046895\" xlink:href=\"#mb4c86e9e83\" y=\"167.997621\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"139.202535\" xlink:href=\"#mb4c86e9e83\" y=\"169.026294\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"139.653361\" xlink:href=\"#mb4c86e9e83\" y=\"158.150304\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"140.240027\" xlink:href=\"#mb4c86e9e83\" y=\"165.009547\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"140.587373\" xlink:href=\"#mb4c86e9e83\" y=\"168.387385\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"142.650408\" xlink:href=\"#mb4c86e9e83\" y=\"170.468654\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"142.65298\" xlink:href=\"#mb4c86e9e83\" y=\"166.979946\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"142.695017\" xlink:href=\"#mb4c86e9e83\" y=\"173.903093\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"142.957411\" xlink:href=\"#mb4c86e9e83\" y=\"155.55059\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"143.265058\" xlink:href=\"#mb4c86e9e83\" y=\"158.898612\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"143.426944\" xlink:href=\"#mb4c86e9e83\" y=\"162.916493\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"144.017659\" xlink:href=\"#mb4c86e9e83\" y=\"171.184519\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"144.100463\" xlink:href=\"#mb4c86e9e83\" y=\"161.417438\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"144.326791\" xlink:href=\"#mb4c86e9e83\" y=\"166.122856\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"144.50482\" xlink:href=\"#mb4c86e9e83\" y=\"165.411698\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"145.430543\" xlink:href=\"#mb4c86e9e83\" y=\"167.053658\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"146.202257\" xlink:href=\"#mb4c86e9e83\" y=\"160.602487\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"146.283351\" xlink:href=\"#mb4c86e9e83\" y=\"163.522282\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"146.959574\" xlink:href=\"#mb4c86e9e83\" y=\"166.973126\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"147.212271\" xlink:href=\"#mb4c86e9e83\" y=\"161.122086\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"147.291037\" xlink:href=\"#mb4c86e9e83\" y=\"162.319261\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"147.76745\" xlink:href=\"#mb4c86e9e83\" y=\"170.29096\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"147.933104\" xlink:href=\"#mb4c86e9e83\" y=\"160.395158\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"148.143148\" xlink:href=\"#mb4c86e9e83\" y=\"153.518252\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"148.613607\" xlink:href=\"#mb4c86e9e83\" y=\"167.995984\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"149.298686\" xlink:href=\"#mb4c86e9e83\" y=\"150.184939\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"149.401265\" xlink:href=\"#mb4c86e9e83\" y=\"156.857986\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"149.478263\" xlink:href=\"#mb4c86e9e83\" y=\"161.979813\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"151.250862\" xlink:href=\"#mb4c86e9e83\" y=\"166.693969\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"152.240953\" xlink:href=\"#mb4c86e9e83\" y=\"162.473641\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"152.566723\" xlink:href=\"#mb4c86e9e83\" y=\"161.592577\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"153.066623\" xlink:href=\"#mb4c86e9e83\" y=\"178.994493\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"153.593124\" xlink:href=\"#mb4c86e9e83\" y=\"160.290886\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"153.835819\" xlink:href=\"#mb4c86e9e83\" y=\"174.580159\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"154.153614\" xlink:href=\"#mb4c86e9e83\" y=\"172.554829\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"155.275156\" xlink:href=\"#mb4c86e9e83\" y=\"184.036977\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"155.37844\" xlink:href=\"#mb4c86e9e83\" y=\"181.316244\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"155.378996\" xlink:href=\"#mb4c86e9e83\" y=\"178.410258\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"155.380541\" xlink:href=\"#mb4c86e9e83\" y=\"170.288531\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"155.505706\" xlink:href=\"#mb4c86e9e83\" y=\"177.721793\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"155.668571\" xlink:href=\"#mb4c86e9e83\" y=\"177.830668\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"157.037884\" xlink:href=\"#mb4c86e9e83\" y=\"194.222851\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"158.541906\" xlink:href=\"#mb4c86e9e83\" y=\"174.690979\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"159.272531\" xlink:href=\"#mb4c86e9e83\" y=\"199.893703\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"160.916736\" xlink:href=\"#mb4c86e9e83\" y=\"182.365716\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"160.920592\" xlink:href=\"#mb4c86e9e83\" y=\"195.728277\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"160.958559\" xlink:href=\"#mb4c86e9e83\" y=\"192.242037\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"161.006285\" xlink:href=\"#mb4c86e9e83\" y=\"182.599698\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"161.377431\" xlink:href=\"#mb4c86e9e83\" y=\"172.236106\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"161.594945\" xlink:href=\"#mb4c86e9e83\" y=\"203.373493\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"161.976459\" xlink:href=\"#mb4c86e9e83\" y=\"171.916457\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"162.103484\" xlink:href=\"#mb4c86e9e83\" y=\"159.585996\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"163.107782\" xlink:href=\"#mb4c86e9e83\" y=\"187.082821\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"163.545433\" xlink:href=\"#mb4c86e9e83\" y=\"165.04616\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"163.661213\" xlink:href=\"#mb4c86e9e83\" y=\"199.636101\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"163.745172\" xlink:href=\"#mb4c86e9e83\" y=\"196.313173\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"163.955762\" xlink:href=\"#mb4c86e9e83\" y=\"185.549513\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"164.518321\" xlink:href=\"#mb4c86e9e83\" y=\"183.413399\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"165.059604\" xlink:href=\"#mb4c86e9e83\" y=\"185.547997\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"165.107562\" xlink:href=\"#mb4c86e9e83\" y=\"199.691529\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"165.242656\" xlink:href=\"#mb4c86e9e83\" y=\"191.100591\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"165.378245\" xlink:href=\"#mb4c86e9e83\" y=\"200.51148\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"165.39379\" xlink:href=\"#mb4c86e9e83\" y=\"190.44393\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"166.319376\" xlink:href=\"#mb4c86e9e83\" y=\"216.168583\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"166.965411\" xlink:href=\"#mb4c86e9e83\" y=\"218.735293\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"166.966078\" xlink:href=\"#mb4c86e9e83\" y=\"208.346274\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"167.657196\" xlink:href=\"#mb4c86e9e83\" y=\"178.255803\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"168.101636\" xlink:href=\"#mb4c86e9e83\" y=\"189.151553\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"170.414111\" xlink:href=\"#mb4c86e9e83\" y=\"192.154067\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"170.757687\" xlink:href=\"#mb4c86e9e83\" y=\"214.827244\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"170.910977\" xlink:href=\"#mb4c86e9e83\" y=\"202.786382\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"171.62915\" xlink:href=\"#mb4c86e9e83\" y=\"181.026774\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"172.356812\" xlink:href=\"#mb4c86e9e83\" y=\"200.14794\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"172.517261\" xlink:href=\"#mb4c86e9e83\" y=\"218.298258\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"173.627679\" xlink:href=\"#mb4c86e9e83\" y=\"224.988724\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"174.375316\" xlink:href=\"#mb4c86e9e83\" y=\"207.101611\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"174.699069\" xlink:href=\"#mb4c86e9e83\" y=\"236.278244\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"175.943788\" xlink:href=\"#mb4c86e9e83\" y=\"202.256849\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"175.950843\" xlink:href=\"#mb4c86e9e83\" y=\"203.643941\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"176.516885\" xlink:href=\"#mb4c86e9e83\" y=\"208.379746\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"176.942296\" xlink:href=\"#mb4c86e9e83\" y=\"224.907638\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"177.442621\" xlink:href=\"#mb4c86e9e83\" y=\"244.264005\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"177.512185\" xlink:href=\"#mb4c86e9e83\" y=\"209.037154\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"178.10168\" xlink:href=\"#mb4c86e9e83\" y=\"189.486972\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"178.344361\" xlink:href=\"#mb4c86e9e83\" y=\"212.541431\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"178.658196\" xlink:href=\"#mb4c86e9e83\" y=\"217.944345\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"178.704001\" xlink:href=\"#mb4c86e9e83\" y=\"215.873781\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"178.903608\" xlink:href=\"#mb4c86e9e83\" y=\"231.927595\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"179.116839\" xlink:href=\"#mb4c86e9e83\" y=\"237.387808\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"179.151952\" xlink:href=\"#mb4c86e9e83\" y=\"202.503378\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"179.153835\" xlink:href=\"#mb4c86e9e83\" y=\"232.158927\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"179.322723\" xlink:href=\"#mb4c86e9e83\" y=\"221.646342\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"179.630201\" xlink:href=\"#mb4c86e9e83\" y=\"248.57176\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"180.827573\" xlink:href=\"#mb4c86e9e83\" y=\"232.965189\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"180.993086\" xlink:href=\"#mb4c86e9e83\" y=\"228.853839\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"182.754254\" xlink:href=\"#mb4c86e9e83\" y=\"245.830544\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"182.934554\" xlink:href=\"#mb4c86e9e83\" y=\"236.497868\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"183.269295\" xlink:href=\"#mb4c86e9e83\" y=\"247.655078\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"183.451127\" xlink:href=\"#mb4c86e9e83\" y=\"228.968282\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"183.954064\" xlink:href=\"#mb4c86e9e83\" y=\"212.208585\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"184.95035\" xlink:href=\"#mb4c86e9e83\" y=\"221.123278\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"185.252166\" xlink:href=\"#mb4c86e9e83\" y=\"272.443297\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"185.400677\" xlink:href=\"#mb4c86e9e83\" y=\"247.569472\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"186.173672\" xlink:href=\"#mb4c86e9e83\" y=\"226.944247\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"186.80719\" xlink:href=\"#mb4c86e9e83\" y=\"229.902576\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"186.980535\" xlink:href=\"#mb4c86e9e83\" y=\"245.658409\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"187.723715\" xlink:href=\"#mb4c86e9e83\" y=\"233.194693\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"188.042921\" xlink:href=\"#mb4c86e9e83\" y=\"273.583336\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"188.076199\" xlink:href=\"#mb4c86e9e83\" y=\"237.154156\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"188.340442\" xlink:href=\"#mb4c86e9e83\" y=\"273.36901\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"189.52207\" xlink:href=\"#mb4c86e9e83\" y=\"241.484016\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"189.942036\" xlink:href=\"#mb4c86e9e83\" y=\"233.366667\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"190.830993\" xlink:href=\"#mb4c86e9e83\" y=\"243.074135\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"191.587912\" xlink:href=\"#mb4c86e9e83\" y=\"222.038455\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"191.98895\" xlink:href=\"#mb4c86e9e83\" y=\"289.781643\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"192.216683\" xlink:href=\"#mb4c86e9e83\" y=\"239.186179\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"192.726532\" xlink:href=\"#mb4c86e9e83\" y=\"237.327461\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"194.171239\" xlink:href=\"#mb4c86e9e83\" y=\"249.775439\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"194.70974\" xlink:href=\"#mb4c86e9e83\" y=\"256.278656\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"196.135406\" xlink:href=\"#mb4c86e9e83\" y=\"245.29744\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"196.431777\" xlink:href=\"#mb4c86e9e83\" y=\"244.452402\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"196.59795\" xlink:href=\"#mb4c86e9e83\" y=\"262.306857\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"197.534198\" xlink:href=\"#mb4c86e9e83\" y=\"283.208907\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"197.588684\" xlink:href=\"#mb4c86e9e83\" y=\"262.909289\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"198.553023\" xlink:href=\"#mb4c86e9e83\" y=\"239.038883\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"199.883861\" xlink:href=\"#mb4c86e9e83\" y=\"292.489873\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"199.981888\" xlink:href=\"#mb4c86e9e83\" y=\"281.390235\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"200.415619\" xlink:href=\"#mb4c86e9e83\" y=\"248.412349\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"201.252339\" xlink:href=\"#mb4c86e9e83\" y=\"247.342216\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"202.059152\" xlink:href=\"#mb4c86e9e83\" y=\"225.079482\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"202.133577\" xlink:href=\"#mb4c86e9e83\" y=\"261.113965\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"202.433364\" xlink:href=\"#mb4c86e9e83\" y=\"230.174392\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"202.885521\" xlink:href=\"#mb4c86e9e83\" y=\"262.791023\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"202.921373\" xlink:href=\"#mb4c86e9e83\" y=\"266.236348\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"203.479975\" xlink:href=\"#mb4c86e9e83\" y=\"233.066221\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"204.01542\" xlink:href=\"#mb4c86e9e83\" y=\"270.06303\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"204.085781\" xlink:href=\"#mb4c86e9e83\" y=\"261.828263\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"206.304903\" xlink:href=\"#mb4c86e9e83\" y=\"243.673986\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"206.411754\" xlink:href=\"#mb4c86e9e83\" y=\"251.380454\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"206.587147\" xlink:href=\"#mb4c86e9e83\" y=\"244.604096\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"206.715842\" xlink:href=\"#mb4c86e9e83\" y=\"244.973676\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"206.8884\" xlink:href=\"#mb4c86e9e83\" y=\"297.614612\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"206.994968\" xlink:href=\"#mb4c86e9e83\" y=\"267.166888\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"207.132421\" xlink:href=\"#mb4c86e9e83\" y=\"256.721075\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"208.815948\" xlink:href=\"#mb4c86e9e83\" y=\"274.98664\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"208.838761\" xlink:href=\"#mb4c86e9e83\" y=\"266.994694\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"209.741088\" xlink:href=\"#mb4c86e9e83\" y=\"305.963877\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"210.538369\" xlink:href=\"#mb4c86e9e83\" y=\"265.757541\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"210.63609\" xlink:href=\"#mb4c86e9e83\" y=\"259.508299\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"210.683142\" xlink:href=\"#mb4c86e9e83\" y=\"238.469521\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"211.867795\" xlink:href=\"#mb4c86e9e83\" y=\"239.285726\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"211.933625\" xlink:href=\"#mb4c86e9e83\" y=\"261.373239\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"213.773243\" xlink:href=\"#mb4c86e9e83\" y=\"249.930214\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"213.941313\" xlink:href=\"#mb4c86e9e83\" y=\"296.768927\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"216.116917\" xlink:href=\"#mb4c86e9e83\" y=\"279.437111\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"216.749161\" xlink:href=\"#mb4c86e9e83\" y=\"232.545618\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"216.936899\" xlink:href=\"#mb4c86e9e83\" y=\"274.86515\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"217.101363\" xlink:href=\"#mb4c86e9e83\" y=\"245.985628\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"217.118754\" xlink:href=\"#mb4c86e9e83\" y=\"238.771986\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"218.179702\" xlink:href=\"#mb4c86e9e83\" y=\"252.753172\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"218.427131\" xlink:href=\"#mb4c86e9e83\" y=\"254.590702\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"218.730841\" xlink:href=\"#mb4c86e9e83\" y=\"278.539057\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"219.205572\" xlink:href=\"#mb4c86e9e83\" y=\"243.146732\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"219.381448\" xlink:href=\"#mb4c86e9e83\" y=\"245.441692\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"219.877249\" xlink:href=\"#mb4c86e9e83\" y=\"249.616383\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"220.888905\" xlink:href=\"#mb4c86e9e83\" y=\"256.444957\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"220.926001\" xlink:href=\"#mb4c86e9e83\" y=\"287.063597\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"220.952035\" xlink:href=\"#mb4c86e9e83\" y=\"253.032234\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"221.43596\" xlink:href=\"#mb4c86e9e83\" y=\"273.012544\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"221.439274\" xlink:href=\"#mb4c86e9e83\" y=\"263.50687\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"222.230833\" xlink:href=\"#mb4c86e9e83\" y=\"259.340504\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"224.101681\" xlink:href=\"#mb4c86e9e83\" y=\"262.257178\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"224.439039\" xlink:href=\"#mb4c86e9e83\" y=\"228.051119\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"225.039589\" xlink:href=\"#mb4c86e9e83\" y=\"259.00353\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"225.657749\" xlink:href=\"#mb4c86e9e83\" y=\"235.36465\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"225.989744\" xlink:href=\"#mb4c86e9e83\" y=\"215.158509\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"226.663752\" xlink:href=\"#mb4c86e9e83\" y=\"230.439281\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"227.993369\" xlink:href=\"#mb4c86e9e83\" y=\"220.734844\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"228.0745\" xlink:href=\"#mb4c86e9e83\" y=\"262.032402\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"230.797904\" xlink:href=\"#mb4c86e9e83\" y=\"237.864303\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"231.325137\" xlink:href=\"#mb4c86e9e83\" y=\"229.674482\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"232.309408\" xlink:href=\"#mb4c86e9e83\" y=\"245.480853\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"233.502284\" xlink:href=\"#mb4c86e9e83\" y=\"188.73799\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"233.655209\" xlink:href=\"#mb4c86e9e83\" y=\"212.769783\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"233.764451\" xlink:href=\"#mb4c86e9e83\" y=\"170.807854\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"233.861859\" xlink:href=\"#mb4c86e9e83\" y=\"187.113671\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"233.946279\" xlink:href=\"#mb4c86e9e83\" y=\"206.523481\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"234.679745\" xlink:href=\"#mb4c86e9e83\" y=\"232.700411\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"235.418124\" xlink:href=\"#mb4c86e9e83\" y=\"241.967458\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"235.511075\" xlink:href=\"#mb4c86e9e83\" y=\"225.984133\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"236.404175\" xlink:href=\"#mb4c86e9e83\" y=\"246.832338\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"236.823294\" xlink:href=\"#mb4c86e9e83\" y=\"203.885635\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"237.085892\" xlink:href=\"#mb4c86e9e83\" y=\"173.457871\"/>\n     <use style=\"fill-opacity:0.5;stroke:#000000;stroke-opacity:0.5;stroke-width:1.5;\" x=\"237.842704\" xlink:href=\"#mb4c86e9e83\" y=\"208.942867\"/>\n    </g>\n   </g>\n   <g id=\"PolyCollection_1\">\n    <path clip-path=\"url(#pebacec704d)\" d=\"M 10.7 251.835158 \nL 10.7 271.662146 \nL 12.382412 270.785445 \nL 14.064824 270.793574 \nL 15.747236 268.751735 \nL 17.429648 265.877712 \nL 19.11206 265.475634 \nL 20.794472 264.572387 \nL 22.476884 261.041532 \nL 24.159296 260.758729 \nL 25.841709 260.555508 \nL 27.524121 259.28444 \nL 29.206533 256.592642 \nL 30.888945 255.67138 \nL 32.571357 256.055669 \nL 34.253769 252.513563 \nL 35.936181 253.050844 \nL 37.618593 251.47161 \nL 39.301005 250.184123 \nL 40.983417 246.229303 \nL 42.665829 245.524144 \nL 44.348241 244.019123 \nL 46.030653 244.04758 \nL 47.713065 240.412438 \nL 49.395477 240.032635 \nL 51.077889 238.924067 \nL 52.760302 236.151789 \nL 54.442714 236.133292 \nL 56.125126 233.938268 \nL 57.807538 233.165273 \nL 59.48995 230.736826 \nL 61.172362 229.493096 \nL 62.854774 227.90164 \nL 64.537186 226.775613 \nL 66.219598 225.507528 \nL 67.90201 223.003012 \nL 69.584422 222.233902 \nL 71.266834 219.744217 \nL 72.949246 218.555471 \nL 74.631658 216.834566 \nL 76.31407 215.91507 \nL 77.996482 214.054409 \nL 79.678894 212.929789 \nL 81.361307 211.051369 \nL 83.043719 208.458324 \nL 84.726131 207.798049 \nL 86.408543 206.218553 \nL 88.090955 204.201699 \nL 89.773367 202.819578 \nL 91.455779 201.278617 \nL 93.138191 199.258386 \nL 94.820603 198.126447 \nL 96.503015 196.316563 \nL 98.185427 194.711047 \nL 99.867839 193.62335 \nL 101.550251 191.443299 \nL 103.232663 189.873668 \nL 104.915075 188.65402 \nL 106.597487 187.045566 \nL 108.279899 185.20454 \nL 109.962312 183.453538 \nL 111.644724 181.984945 \nL 113.327136 180.336565 \nL 115.009548 178.628853 \nL 116.69196 177.01267 \nL 118.374372 175.30582 \nL 120.056784 173.600515 \nL 121.739196 172.070284 \nL 123.421608 170.08035 \nL 125.10402 168.628539 \nL 126.786432 166.971397 \nL 128.468844 165.466118 \nL 130.151256 163.883618 \nL 131.833668 162.421792 \nL 133.51608 161.281994 \nL 135.198492 160.453393 \nL 136.880905 159.958125 \nL 138.563317 159.558206 \nL 140.245729 159.462285 \nL 141.928141 159.243517 \nL 143.610553 159.327068 \nL 145.292965 159.417801 \nL 146.975377 159.479025 \nL 148.657789 159.576579 \nL 150.340201 159.88641 \nL 152.022613 161.115503 \nL 153.705025 163.050641 \nL 155.387437 164.983829 \nL 157.069849 167.109208 \nL 158.752261 169.326361 \nL 160.434673 171.605952 \nL 162.117085 174.149788 \nL 163.799497 176.845243 \nL 165.48191 179.56124 \nL 167.164322 182.262191 \nL 168.846734 185.009426 \nL 170.529146 187.371172 \nL 172.211558 190.060919 \nL 173.89397 192.862297 \nL 175.576382 195.652894 \nL 177.258794 198.412128 \nL 178.941206 200.94261 \nL 180.623618 203.949922 \nL 182.30603 207.1828 \nL 183.988442 209.637879 \nL 185.670854 211.851989 \nL 187.353266 214.145951 \nL 189.035678 216.848365 \nL 190.71809 219.842822 \nL 192.400503 222.085065 \nL 194.082915 225.15939 \nL 195.765327 228.291208 \nL 197.447739 230.398326 \nL 199.130151 231.947874 \nL 200.812563 234.361646 \nL 202.494975 235.431348 \nL 204.177387 236.309702 \nL 205.859799 237.097897 \nL 207.542211 238.646042 \nL 209.224623 239.631121 \nL 210.907035 238.738014 \nL 212.589447 238.023386 \nL 214.271859 236.372418 \nL 215.954271 233.233254 \nL 217.636683 230.873309 \nL 219.319095 226.765489 \nL 221.001508 224.204053 \nL 222.68392 218.540034 \nL 224.366332 216.129362 \nL 226.048744 213.58506 \nL 227.731156 209.848636 \nL 229.413568 205.830098 \nL 231.09598 201.992881 \nL 232.778392 198.80347 \nL 234.460804 193.15048 \nL 236.143216 190.739494 \nL 237.825628 186.279163 \nL 239.50804 181.997012 \nL 241.190452 179.798877 \nL 242.872864 175.026605 \nL 244.555276 169.437267 \nL 246.237688 168.067149 \nL 247.920101 165.821504 \nL 249.602513 160.337221 \nL 251.284925 157.599508 \nL 252.967337 151.845364 \nL 254.649749 148.46032 \nL 256.332161 145.228784 \nL 258.014573 140.368013 \nL 259.696985 135.898449 \nL 261.379397 135.316783 \nL 263.061809 129.610973 \nL 264.744221 124.905292 \nL 266.426633 119.255174 \nL 268.109045 116.188715 \nL 269.791457 110.83823 \nL 271.473869 103.376226 \nL 273.156281 101.021905 \nL 274.838693 96.497188 \nL 276.521106 97.519441 \nL 278.203518 87.901846 \nL 279.88593 85.769519 \nL 281.568342 80.389636 \nL 283.250754 71.323659 \nL 284.933166 67.551175 \nL 286.615578 64.733851 \nL 288.29799 57.009725 \nL 289.980402 56.599352 \nL 291.662814 47.873759 \nL 293.345226 46.748274 \nL 295.027638 34.594017 \nL 296.71005 31.358356 \nL 298.392462 31.882491 \nL 300.074874 21.105211 \nL 301.757286 10.670143 \nL 303.439698 7.082137 \nL 305.122111 4.337699 \nL 306.804523 1.605576 \nL 308.486935 -11.012278 \nL 310.169347 -15.813205 \nL 311.851759 -13.872659 \nL 313.534171 -24.61674 \nL 315.216583 -32.197684 \nL 316.898995 -37.252725 \nL 318.581407 -42.966482 \nL 320.263819 -54.211853 \nL 321.946231 -66.540521 \nL 323.628643 -62.435386 \nL 325.311055 -86.454428 \nL 326.993467 -93.408423 \nL 328.675879 -89.422624 \nL 330.358291 -98.107418 \nL 332.040704 -98.29628 \nL 333.723116 -110.36874 \nL 335.405528 -125.918 \nL 337.08794 -132.524826 \nL 338.770352 -140.07105 \nL 340.452764 -149.29095 \nL 342.135176 -169.227769 \nL 343.817588 -161.844852 \nL 345.5 -177.727908 \nL 345.5 -202.977848 \nL 345.5 -202.977848 \nL 343.817588 -187.388758 \nL 342.135176 -195.351369 \nL 340.452764 -175.665207 \nL 338.770352 -164.878454 \nL 337.08794 -158.84381 \nL 335.405528 -152.382271 \nL 333.723116 -136.976602 \nL 332.040704 -124.947562 \nL 330.358291 -123.702632 \nL 328.675879 -116.425459 \nL 326.993467 -119.99822 \nL 325.311055 -111.342637 \nL 323.628643 -89.956914 \nL 321.946231 -93.046252 \nL 320.263819 -81.86957 \nL 318.581407 -70.612017 \nL 316.898995 -63.751495 \nL 315.216583 -58.410483 \nL 313.534171 -49.706964 \nL 311.851759 -40.707759 \nL 310.169347 -41.992145 \nL 308.486935 -38.347874 \nL 306.804523 -24.928309 \nL 305.122111 -20.185805 \nL 303.439698 -18.755251 \nL 301.757286 -13.269724 \nL 300.074874 -3.955976 \nL 298.392462 6.486824 \nL 296.71005 5.887589 \nL 295.027638 9.22256 \nL 293.345226 22.564008 \nL 291.662814 24.561324 \nL 289.980402 33.531406 \nL 288.29799 34.189956 \nL 286.615578 40.086561 \nL 284.933166 44.009035 \nL 283.250754 47.813914 \nL 281.568342 56.838619 \nL 279.88593 62.24855 \nL 278.203518 65.925194 \nL 276.521106 75.511439 \nL 274.838693 73.983767 \nL 273.156281 79.707063 \nL 271.473869 81.707045 \nL 269.791457 90.994786 \nL 268.109045 96.623133 \nL 266.426633 99.493306 \nL 264.744221 105.621291 \nL 263.061809 110.594789 \nL 261.379397 117.909452 \nL 259.696985 118.782655 \nL 258.014573 121.810062 \nL 256.332161 129.051728 \nL 254.649749 132.131599 \nL 252.967337 136.50606 \nL 251.284925 141.778663 \nL 249.602513 145.342613 \nL 247.920101 151.195593 \nL 246.237688 153.817595 \nL 244.555276 156.030121 \nL 242.872864 161.555575 \nL 241.190452 167.438907 \nL 239.50804 169.781534 \nL 237.825628 174.618251 \nL 236.143216 179.942566 \nL 234.460804 181.867006 \nL 232.778392 188.742946 \nL 231.09598 192.223529 \nL 229.413568 196.564607 \nL 227.731156 200.576738 \nL 226.048744 205.465209 \nL 224.366332 208.224968 \nL 222.68392 210.962219 \nL 221.001508 216.888884 \nL 219.319095 219.793261 \nL 217.636683 223.98724 \nL 215.954271 226.926847 \nL 214.271859 230.152505 \nL 212.589447 231.990876 \nL 210.907035 233.30329 \nL 209.224623 234.406312 \nL 207.542211 233.656208 \nL 205.859799 232.277047 \nL 204.177387 231.816788 \nL 202.494975 231.185599 \nL 200.812563 230.089571 \nL 199.130151 228.238949 \nL 197.447739 226.746389 \nL 195.765327 224.793371 \nL 194.082915 221.960459 \nL 192.400503 219.090533 \nL 190.71809 217.011283 \nL 189.035678 214.160635 \nL 187.353266 211.633252 \nL 185.670854 209.650111 \nL 183.988442 207.566228 \nL 182.30603 205.1348 \nL 180.623618 202.123612 \nL 178.941206 199.290767 \nL 177.258794 196.864341 \nL 175.576382 194.243381 \nL 173.89397 191.753356 \nL 172.211558 188.954409 \nL 170.529146 186.484768 \nL 168.846734 184.215367 \nL 167.164322 181.581595 \nL 165.48191 178.965294 \nL 163.799497 176.377174 \nL 162.117085 173.783035 \nL 160.434673 171.299811 \nL 158.752261 169.084376 \nL 157.069849 166.92753 \nL 155.387437 164.818775 \nL 153.705025 162.917423 \nL 152.022613 160.995548 \nL 150.340201 159.757111 \nL 148.657789 159.440175 \nL 146.975377 159.326011 \nL 145.292965 159.259426 \nL 143.610553 159.160352 \nL 141.928141 159.068113 \nL 140.245729 159.263358 \nL 138.563317 159.340713 \nL 136.880905 159.707153 \nL 135.198492 160.166999 \nL 133.51608 160.982005 \nL 131.833668 162.060464 \nL 130.151256 163.479094 \nL 128.468844 164.943506 \nL 126.786432 166.3669 \nL 125.10402 167.942491 \nL 123.421608 169.183706 \nL 121.739196 170.998131 \nL 120.056784 172.322719 \nL 118.374372 173.831941 \nL 116.69196 175.270729 \nL 115.009548 176.528117 \nL 113.327136 178.166094 \nL 111.644724 179.350849 \nL 109.962312 180.80245 \nL 108.279899 182.28798 \nL 106.597487 183.636825 \nL 104.915075 185.282643 \nL 103.232663 185.79716 \nL 101.550251 187.235447 \nL 99.867839 189.180156 \nL 98.185427 189.479947 \nL 96.503015 191.152648 \nL 94.820603 191.984958 \nL 93.138191 193.469974 \nL 91.455779 195.074118 \nL 89.773367 196.280178 \nL 88.090955 197.381266 \nL 86.408543 198.58308 \nL 84.726131 200.086814 \nL 83.043719 200.601761 \nL 81.361307 202.976108 \nL 79.678894 204.519371 \nL 77.996482 205.456677 \nL 76.31407 206.555906 \nL 74.631658 206.689925 \nL 72.949246 208.546829 \nL 71.266834 208.867055 \nL 69.584422 211.429222 \nL 67.90201 211.529148 \nL 66.219598 213.696421 \nL 64.537186 214.739696 \nL 62.854774 215.872249 \nL 61.172362 217.276998 \nL 59.48995 217.77321 \nL 57.807538 220.208406 \nL 56.125126 220.417266 \nL 54.442714 222.145727 \nL 52.760302 221.378027 \nL 51.077889 224.031394 \nL 49.395477 225.258549 \nL 47.713065 225.250899 \nL 46.030653 228.736791 \nL 44.348241 227.994335 \nL 42.665829 229.41307 \nL 40.983417 229.533719 \nL 39.301005 233.817831 \nL 37.618593 234.214528 \nL 35.936181 235.912533 \nL 34.253769 234.456815 \nL 32.571357 238.156772 \nL 30.888945 235.81265 \nL 29.206533 237.616971 \nL 27.524121 239.883052 \nL 25.841709 240.643433 \nL 24.159296 240.305525 \nL 22.476884 240.94601 \nL 20.794472 244.256784 \nL 19.11206 244.796882 \nL 17.429648 245.785498 \nL 15.747236 248.823398 \nL 14.064824 249.527867 \nL 12.382412 247.924602 \nL 10.7 251.835158 \nz\n\" style=\"fill:#1f77b4;fill-opacity:0.3;stroke:#1f77b4;stroke-opacity:0.3;\"/>\n   </g>\n   <g id=\"PolyCollection_2\">\n    <defs>\n     <path d=\"M 10.7 -82.028693 \nL 10.7 -62.201704 \nL 12.382412 -62.072125 \nL 14.064824 -68.556597 \nL 15.747236 -71.311852 \nL 17.429648 -74.418527 \nL 19.11206 -74.566435 \nL 20.794472 -75.267492 \nL 22.476884 -79.731593 \nL 24.159296 -81.388692 \nL 25.841709 -83.376029 \nL 27.524121 -85.152011 \nL 29.206533 -89.094809 \nL 30.888945 -89.013232 \nL 32.571357 -91.607371 \nL 34.253769 -95.353816 \nL 35.936181 -96.151044 \nL 37.618593 -98.75394 \nL 39.301005 -100.811305 \nL 40.983417 -105.639951 \nL 42.665829 -106.80097 \nL 44.348241 -109.165139 \nL 46.030653 -109.481036 \nL 47.713065 -113.839537 \nL 49.395477 -115.977702 \nL 51.077889 -116.414466 \nL 52.760302 -120.523991 \nL 54.442714 -121.309615 \nL 56.125126 -124.389436 \nL 57.807538 -126.46063 \nL 59.48995 -128.730357 \nL 61.172362 -131.724338 \nL 62.854774 -133.385143 \nL 64.537186 -134.204947 \nL 66.219598 -135.753636 \nL 67.90201 -139.713236 \nL 69.584422 -141.236035 \nL 71.266834 -144.242182 \nL 72.949246 -146.379334 \nL 74.631658 -148.504512 \nL 76.31407 -150.144774 \nL 77.996482 -153.187061 \nL 79.678894 -154.278785 \nL 81.361307 -157.020442 \nL 83.043719 -160.495677 \nL 84.726131 -161.003719 \nL 86.408543 -162.91578 \nL 88.090955 -166.173142 \nL 89.773367 -168.390661 \nL 91.455779 -170.168508 \nL 93.138191 -172.881722 \nL 94.820603 -173.622819 \nL 96.503015 -176.678945 \nL 98.185427 -178.319554 \nL 99.867839 -180.331095 \nL 101.550251 -183.281799 \nL 103.232663 -184.810445 \nL 104.915075 -186.977187 \nL 106.597487 -188.62525 \nL 108.279899 -191.248222 \nL 109.962312 -193.67206 \nL 111.644724 -195.019463 \nL 113.327136 -197.18544 \nL 115.009548 -199.169271 \nL 116.69196 -201.310173 \nL 118.374372 -203.584491 \nL 120.056784 -205.608387 \nL 121.739196 -207.345869 \nL 123.421608 -209.767895 \nL 125.10402 -211.496841 \nL 126.786432 -213.370567 \nL 128.468844 -215.023579 \nL 130.151256 -216.816384 \nL 131.833668 -218.407657 \nL 133.51608 -219.513196 \nL 135.198492 -220.096564 \nL 136.880905 -220.400906 \nL 138.563317 -220.362718 \nL 140.245729 -220.016031 \nL 141.928141 -219.97446 \nL 143.610553 -219.449472 \nL 145.292965 -218.793081 \nL 146.975377 -218.352301 \nL 148.657789 -217.783617 \nL 150.340201 -216.994526 \nL 152.022613 -215.115317 \nL 153.705025 -212.534858 \nL 155.387437 -209.953609 \nL 157.069849 -207.07431 \nL 158.752261 -204.065198 \nL 160.434673 -200.963086 \nL 162.117085 -197.580676 \nL 163.799497 -193.999085 \nL 165.48191 -190.288514 \nL 167.164322 -186.605636 \nL 168.846734 -182.913746 \nL 170.529146 -179.554222 \nL 172.211558 -175.819341 \nL 173.89397 -172.26901 \nL 175.576382 -168.271741 \nL 177.258794 -164.495228 \nL 178.941206 -161.146313 \nL 180.623618 -156.975944 \nL 182.30603 -152.710048 \nL 183.988442 -149.385013 \nL 185.670854 -146.397649 \nL 187.353266 -143.062653 \nL 189.035678 -139.431348 \nL 190.71809 -135.556755 \nL 192.400503 -132.727436 \nL 194.082915 -128.610768 \nL 195.765327 -124.462542 \nL 197.447739 -121.522181 \nL 199.130151 -119.412215 \nL 200.812563 -115.641543 \nL 202.494975 -113.941756 \nL 204.177387 -112.007522 \nL 205.859799 -110.154302 \nL 207.542211 -107.281281 \nL 209.224623 -105.353265 \nL 210.907035 -105.097713 \nL 212.589447 -104.343217 \nL 214.271859 -104.690617 \nL 215.954271 -107.131308 \nL 217.636683 -107.5947 \nL 219.319095 -110.084534 \nL 221.001508 -110.768413 \nL 222.68392 -115.082771 \nL 224.366332 -115.36743 \nL 226.048744 -116.339713 \nL 227.731156 -116.642299 \nL 229.413568 -119.1492 \nL 231.09598 -120.433872 \nL 232.778392 -121.409779 \nL 234.460804 -124.885152 \nL 236.143216 -125.407618 \nL 237.825628 -127.377128 \nL 239.50804 -128.183432 \nL 241.190452 -128.367778 \nL 242.872864 -130.273715 \nL 244.555276 -133.142313 \nL 246.237688 -130.865349 \nL 247.920101 -130.934176 \nL 249.602513 -133.6974 \nL 251.284925 -132.923425 \nL 252.967337 -136.818731 \nL 254.649749 -137.347294 \nL 256.332161 -138.73669 \nL 258.014573 -138.655868 \nL 259.696985 -140.217693 \nL 261.379397 -137.944094 \nL 263.061809 -139.217955 \nL 264.744221 -142.364862 \nL 266.426633 -141.782068 \nL 268.109045 -143.847666 \nL 269.791457 -143.252496 \nL 271.473869 -147.384006 \nL 273.156281 -145.952034 \nL 274.838693 -143.198644 \nL 276.521106 -140.704399 \nL 278.203518 -148.550147 \nL 279.88593 -144.021405 \nL 281.568342 -144.698708 \nL 283.250754 -147.916771 \nL 284.933166 -148.375791 \nL 286.615578 -147.392306 \nL 288.29799 -151.478231 \nL 289.980402 -146.768503 \nL 291.662814 -150.112517 \nL 293.345226 -143.918784 \nL 295.027638 -151.301546 \nL 296.71005 -147.613216 \nL 298.392462 -139.50762 \nL 300.074874 -148.698 \nL 301.757286 -151.756122 \nL 303.439698 -145.88207 \nL 305.122111 -143.87804 \nL 306.804523 -142.662123 \nL 308.486935 -143.18037 \nL 310.169347 -145.731077 \nL 311.851759 -137.502951 \nL 313.534171 -139.422536 \nL 315.216583 -139.294035 \nL 316.898995 -135.487647 \nL 318.581407 -129.85246 \nL 320.263819 -132.557068 \nL 321.946231 -129.754284 \nL 323.628643 -129.061025 \nL 325.311055 -128.876701 \nL 326.993467 -129.241362 \nL 328.675879 -127.458881 \nL 330.358291 -121.439107 \nL 332.040704 -121.970104 \nL 333.723116 -117.020453 \nL 335.405528 -115.440041 \nL 337.08794 -111.122394 \nL 338.770352 -116.599804 \nL 340.452764 -100.666255 \nL 342.135176 -100.65047 \nL 343.817588 -96.567695 \nL 345.5 -90.650889 \nL 345.5 -115.900851 \nL 345.5 -115.900851 \nL 343.817588 -122.111602 \nL 342.135176 -126.774049 \nL 340.452764 -127.040511 \nL 338.770352 -141.407185 \nL 337.08794 -137.441378 \nL 335.405528 -141.904333 \nL 333.723116 -143.628314 \nL 332.040704 -148.621375 \nL 330.358291 -147.03431 \nL 328.675879 -154.461727 \nL 326.993467 -155.831159 \nL 325.311055 -153.764911 \nL 323.628643 -156.582554 \nL 321.946231 -156.260003 \nL 320.263819 -160.214796 \nL 318.581407 -157.497983 \nL 316.898995 -161.986405 \nL 315.216583 -165.506824 \nL 313.534171 -164.512759 \nL 311.851759 -164.338051 \nL 310.169347 -171.910039 \nL 308.486935 -170.515988 \nL 306.804523 -169.196008 \nL 305.122111 -168.401533 \nL 303.439698 -171.719458 \nL 301.757286 -175.695978 \nL 300.074874 -173.759177 \nL 298.392462 -164.903288 \nL 296.71005 -173.083994 \nL 295.027638 -176.673014 \nL 293.345226 -168.103061 \nL 291.662814 -173.42494 \nL 289.980402 -169.836455 \nL 288.29799 -174.297995 \nL 286.615578 -172.039597 \nL 284.933166 -171.917931 \nL 283.250754 -171.426522 \nL 281.568342 -168.249731 \nL 279.88593 -167.542373 \nL 278.203518 -170.526794 \nL 276.521106 -162.712397 \nL 274.838693 -165.712068 \nL 273.156281 -167.266876 \nL 271.473869 -169.053177 \nL 269.791457 -163.095937 \nL 268.109045 -163.413256 \nL 266.426633 -161.543944 \nL 264.744221 -161.648869 \nL 263.061809 -158.234139 \nL 261.379397 -155.351425 \nL 259.696985 -157.333486 \nL 258.014573 -157.21382 \nL 256.332161 -154.913747 \nL 254.649749 -153.676014 \nL 252.967337 -152.158035 \nL 251.284925 -148.744263 \nL 249.602513 -148.692008 \nL 247.920101 -145.560081 \nL 246.237688 -145.114907 \nL 244.555276 -146.54946 \nL 242.872864 -143.744748 \nL 241.190452 -140.727751 \nL 239.50804 -140.398911 \nL 237.825628 -139.038037 \nL 236.143216 -136.204543 \nL 234.460804 -136.168629 \nL 232.778392 -131.470301 \nL 231.09598 -130.203225 \nL 229.413568 -128.414691 \nL 227.731156 -125.914201 \nL 226.048744 -124.459568 \nL 224.366332 -123.271827 \nL 222.68392 -122.660586 \nL 221.001508 -118.083582 \nL 219.319095 -117.056758 \nL 217.636683 -114.480772 \nL 215.954271 -113.437712 \nL 214.271859 -110.910533 \nL 212.589447 -110.375725 \nL 210.907035 -110.532434 \nL 209.224623 -110.57808 \nL 207.542211 -112.271115 \nL 205.859799 -114.975155 \nL 204.177387 -116.500438 \nL 202.494975 -118.187508 \nL 200.812563 -119.913615 \nL 199.130151 -123.121143 \nL 197.447739 -125.174118 \nL 195.765327 -127.96038 \nL 194.082915 -131.809699 \nL 192.400503 -135.721968 \nL 190.71809 -138.388293 \nL 189.035678 -142.119081 \nL 187.353266 -145.575353 \nL 185.670854 -148.599527 \nL 183.988442 -151.456664 \nL 182.30603 -154.758045 \nL 180.623618 -158.802254 \nL 178.941206 -162.798154 \nL 177.258794 -166.043015 \nL 175.576382 -169.681254 \nL 173.89397 -173.377951 \nL 172.211558 -176.925851 \nL 170.529146 -180.440625 \nL 168.846734 -183.707805 \nL 167.164322 -187.286234 \nL 165.48191 -190.884458 \nL 163.799497 -194.467154 \nL 162.117085 -197.947431 \nL 160.434673 -201.269228 \nL 158.752261 -204.307184 \nL 157.069849 -207.255987 \nL 155.387437 -210.118661 \nL 153.705025 -212.668075 \nL 152.022613 -215.235272 \nL 150.340201 -217.123825 \nL 148.657789 -217.92002 \nL 146.975377 -218.505315 \nL 145.292965 -218.951456 \nL 143.610553 -219.616188 \nL 141.928141 -220.149865 \nL 140.245729 -220.214958 \nL 138.563317 -220.58021 \nL 136.880905 -220.651879 \nL 135.198492 -220.382958 \nL 133.51608 -219.813185 \nL 131.833668 -218.768985 \nL 130.151256 -217.220907 \nL 128.468844 -215.546191 \nL 126.786432 -213.975064 \nL 125.10402 -212.182889 \nL 123.421608 -210.664538 \nL 121.739196 -208.418022 \nL 120.056784 -206.886183 \nL 118.374372 -205.05837 \nL 116.69196 -203.052113 \nL 115.009548 -201.270009 \nL 113.327136 -199.355911 \nL 111.644724 -197.653557 \nL 109.962312 -196.323148 \nL 108.279899 -194.164781 \nL 106.597487 -192.033991 \nL 104.915075 -190.348563 \nL 103.232663 -188.886952 \nL 101.550251 -187.489652 \nL 99.867839 -184.774289 \nL 98.185427 -183.550654 \nL 96.503015 -181.842861 \nL 94.820603 -179.764308 \nL 93.138191 -178.670135 \nL 91.455779 -176.373008 \nL 89.773367 -174.930062 \nL 88.090955 -172.993574 \nL 86.408543 -170.551253 \nL 84.726131 -168.714952 \nL 83.043719 -168.352241 \nL 81.361307 -165.095702 \nL 79.678894 -162.689203 \nL 77.996482 -161.784794 \nL 76.31407 -159.503936 \nL 74.631658 -158.649155 \nL 72.949246 -156.387975 \nL 71.266834 -155.119343 \nL 69.584422 -152.040717 \nL 67.90201 -151.187101 \nL 66.219598 -147.564739 \nL 64.537186 -146.240865 \nL 62.854774 -145.414535 \nL 61.172362 -143.940439 \nL 59.48995 -141.69397 \nL 57.807538 -139.417496 \nL 56.125126 -137.910438 \nL 54.442714 -135.297182 \nL 52.760302 -135.29775 \nL 51.077889 -131.307137 \nL 49.395477 -130.751791 \nL 47.713065 -129.001071 \nL 46.030653 -124.791828 \nL 44.348241 -125.189924 \nL 42.665829 -122.912041 \nL 40.983417 -122.335535 \nL 39.301005 -117.177598 \nL 37.618593 -116.011022 \nL 35.936181 -113.289352 \nL 34.253769 -113.410566 \nL 32.571357 -109.506266 \nL 30.888945 -108.87196 \nL 29.206533 -108.070474 \nL 27.524121 -104.553404 \nL 25.841709 -103.288107 \nL 24.159296 -101.841896 \nL 22.476884 -99.827115 \nL 20.794472 -95.583092 \nL 19.11206 -95.245187 \nL 17.429648 -94.510744 \nL 15.747236 -91.240188 \nL 14.064824 -89.822301 \nL 12.382412 -84.932968 \nL 10.7 -82.028693 \nz\n\" id=\"meb971f193d\" style=\"stroke:#1f77b4;stroke-opacity:0.3;\"/>\n    </defs>\n    <g clip-path=\"url(#pebacec704d)\">\n     <use style=\"fill:#1f77b4;fill-opacity:0.3;stroke:#1f77b4;stroke-opacity:0.3;\" x=\"0\" xlink:href=\"#meb971f193d\" y=\"397.0625\"/>\n    </g>\n   </g>\n   <g id=\"PolyCollection_3\">\n    <path clip-path=\"url(#pebacec704d)\" d=\"M 10.7 271.662146 \nL 10.7 315.033807 \nL 12.382412 312.129532 \nL 14.064824 307.240199 \nL 15.747236 305.822312 \nL 17.429648 302.551756 \nL 19.11206 301.817313 \nL 20.794472 301.479408 \nL 22.476884 297.235385 \nL 24.159296 295.220604 \nL 25.841709 293.774393 \nL 27.524121 292.509096 \nL 29.206533 288.992026 \nL 30.888945 288.19054 \nL 32.571357 287.556234 \nL 34.253769 283.651934 \nL 35.936181 283.773148 \nL 37.618593 281.051478 \nL 39.301005 279.884902 \nL 40.983417 274.726965 \nL 42.665829 274.150459 \nL 44.348241 271.872576 \nL 46.030653 272.270672 \nL 47.713065 268.061429 \nL 49.395477 266.310709 \nL 51.077889 265.755363 \nL 52.760302 261.76475 \nL 54.442714 261.765318 \nL 56.125126 259.152062 \nL 57.807538 257.645004 \nL 59.48995 255.36853 \nL 61.172362 253.122061 \nL 62.854774 251.647965 \nL 64.537186 250.821635 \nL 66.219598 249.497761 \nL 67.90201 245.875399 \nL 69.584422 245.021783 \nL 71.266834 241.943157 \nL 72.949246 240.674525 \nL 74.631658 238.413345 \nL 76.31407 237.558564 \nL 77.996482 235.277706 \nL 79.678894 234.373297 \nL 81.361307 231.966798 \nL 83.043719 228.710259 \nL 84.726131 228.347548 \nL 86.408543 226.511247 \nL 88.090955 224.068926 \nL 89.773367 222.132438 \nL 91.455779 220.689492 \nL 93.138191 218.392365 \nL 94.820603 217.298192 \nL 96.503015 215.219639 \nL 98.185427 213.511846 \nL 99.867839 212.288211 \nL 101.550251 209.572848 \nL 103.232663 208.175548 \nL 104.915075 206.713937 \nL 106.597487 205.028509 \nL 108.279899 202.897719 \nL 109.962312 200.739352 \nL 111.644724 199.408943 \nL 113.327136 197.706589 \nL 115.009548 195.792491 \nL 116.69196 194.010387 \nL 118.374372 192.00413 \nL 120.056784 190.176317 \nL 121.739196 188.644478 \nL 123.421608 186.397962 \nL 125.10402 184.879611 \nL 126.786432 183.087436 \nL 128.468844 181.516309 \nL 130.151256 179.841593 \nL 131.833668 178.293515 \nL 133.51608 177.249315 \nL 135.198492 176.679542 \nL 136.880905 176.410621 \nL 138.563317 176.48229 \nL 140.245729 176.847542 \nL 141.928141 176.912635 \nL 143.610553 177.446312 \nL 145.292965 178.111044 \nL 146.975377 178.557185 \nL 148.657789 179.14248 \nL 150.340201 179.938675 \nL 152.022613 181.827228 \nL 153.705025 184.394425 \nL 155.387437 186.943839 \nL 157.069849 189.806513 \nL 158.752261 192.755316 \nL 160.434673 195.793272 \nL 162.117085 199.115069 \nL 163.799497 202.595346 \nL 165.48191 206.178042 \nL 167.164322 209.776266 \nL 168.846734 213.354695 \nL 170.529146 216.621875 \nL 172.211558 220.136649 \nL 173.89397 223.684549 \nL 175.576382 227.381246 \nL 177.258794 231.019485 \nL 178.941206 234.264346 \nL 180.623618 238.260246 \nL 182.30603 242.304455 \nL 183.988442 245.605836 \nL 185.670854 248.462973 \nL 187.353266 251.487147 \nL 189.035678 254.943419 \nL 190.71809 258.674207 \nL 192.400503 261.340532 \nL 194.082915 265.252801 \nL 195.765327 269.10212 \nL 197.447739 271.888382 \nL 199.130151 273.941357 \nL 200.812563 277.148885 \nL 202.494975 278.874992 \nL 204.177387 280.562062 \nL 205.859799 282.087345 \nL 207.542211 284.791385 \nL 209.224623 286.48442 \nL 210.907035 286.530066 \nL 212.589447 286.686775 \nL 214.271859 286.151967 \nL 215.954271 283.624788 \nL 217.636683 282.581728 \nL 219.319095 280.005742 \nL 221.001508 278.978918 \nL 222.68392 274.401914 \nL 224.366332 273.790673 \nL 226.048744 272.602932 \nL 227.731156 271.148299 \nL 229.413568 268.647809 \nL 231.09598 266.859275 \nL 232.778392 265.592199 \nL 234.460804 260.893871 \nL 236.143216 260.857957 \nL 237.825628 258.024463 \nL 239.50804 256.663589 \nL 241.190452 256.334749 \nL 242.872864 253.317752 \nL 244.555276 250.51304 \nL 246.237688 251.947593 \nL 247.920101 251.502419 \nL 249.602513 248.370492 \nL 251.284925 248.318237 \nL 252.967337 244.904465 \nL 254.649749 243.386486 \nL 256.332161 242.148753 \nL 258.014573 239.84868 \nL 259.696985 239.729014 \nL 261.379397 241.711075 \nL 263.061809 238.828361 \nL 264.744221 235.413631 \nL 266.426633 235.518556 \nL 268.109045 233.649244 \nL 269.791457 233.966563 \nL 271.473869 228.009323 \nL 273.156281 229.795624 \nL 274.838693 231.350432 \nL 276.521106 234.350103 \nL 278.203518 226.535706 \nL 279.88593 229.520127 \nL 281.568342 228.812769 \nL 283.250754 225.635978 \nL 284.933166 225.144569 \nL 286.615578 225.022903 \nL 288.29799 222.764505 \nL 289.980402 227.226045 \nL 291.662814 223.63756 \nL 293.345226 228.959439 \nL 295.027638 220.389486 \nL 296.71005 223.978506 \nL 298.392462 232.159212 \nL 300.074874 223.303323 \nL 301.757286 221.366522 \nL 303.439698 225.343042 \nL 305.122111 228.660967 \nL 306.804523 227.866492 \nL 308.486935 226.546512 \nL 310.169347 225.152461 \nL 311.851759 232.724449 \nL 313.534171 232.549741 \nL 315.216583 231.555676 \nL 316.898995 235.076095 \nL 318.581407 239.564517 \nL 320.263819 236.847704 \nL 321.946231 240.802497 \nL 323.628643 240.479946 \nL 325.311055 243.297589 \nL 326.993467 241.231341 \nL 328.675879 242.600773 \nL 330.358291 250.02819 \nL 332.040704 248.441125 \nL 333.723116 253.434186 \nL 335.405528 255.158167 \nL 337.08794 259.621122 \nL 338.770352 255.655315 \nL 340.452764 270.021989 \nL 342.135176 270.288451 \nL 343.817588 274.950898 \nL 345.5 281.161649 \nL 345.5 -177.727908 \nL 345.5 -177.727908 \nL 343.817588 -161.844852 \nL 342.135176 -169.227769 \nL 340.452764 -149.29095 \nL 338.770352 -140.07105 \nL 337.08794 -132.524826 \nL 335.405528 -125.918 \nL 333.723116 -110.36874 \nL 332.040704 -98.29628 \nL 330.358291 -98.107418 \nL 328.675879 -89.422624 \nL 326.993467 -93.408423 \nL 325.311055 -86.454428 \nL 323.628643 -62.435386 \nL 321.946231 -66.540521 \nL 320.263819 -54.211853 \nL 318.581407 -42.966482 \nL 316.898995 -37.252725 \nL 315.216583 -32.197684 \nL 313.534171 -24.61674 \nL 311.851759 -13.872659 \nL 310.169347 -15.813205 \nL 308.486935 -11.012278 \nL 306.804523 1.605576 \nL 305.122111 4.337699 \nL 303.439698 7.082137 \nL 301.757286 10.670143 \nL 300.074874 21.105211 \nL 298.392462 31.882491 \nL 296.71005 31.358356 \nL 295.027638 34.594017 \nL 293.345226 46.748274 \nL 291.662814 47.873759 \nL 289.980402 56.599352 \nL 288.29799 57.009725 \nL 286.615578 64.733851 \nL 284.933166 67.551175 \nL 283.250754 71.323659 \nL 281.568342 80.389636 \nL 279.88593 85.769519 \nL 278.203518 87.901846 \nL 276.521106 97.519441 \nL 274.838693 96.497188 \nL 273.156281 101.021905 \nL 271.473869 103.376226 \nL 269.791457 110.83823 \nL 268.109045 116.188715 \nL 266.426633 119.255174 \nL 264.744221 124.905292 \nL 263.061809 129.610973 \nL 261.379397 135.316783 \nL 259.696985 135.898449 \nL 258.014573 140.368013 \nL 256.332161 145.228784 \nL 254.649749 148.46032 \nL 252.967337 151.845364 \nL 251.284925 157.599508 \nL 249.602513 160.337221 \nL 247.920101 165.821504 \nL 246.237688 168.067149 \nL 244.555276 169.437267 \nL 242.872864 175.026605 \nL 241.190452 179.798877 \nL 239.50804 181.997012 \nL 237.825628 186.279163 \nL 236.143216 190.739494 \nL 234.460804 193.15048 \nL 232.778392 198.80347 \nL 231.09598 201.992881 \nL 229.413568 205.830098 \nL 227.731156 209.848636 \nL 226.048744 213.58506 \nL 224.366332 216.129362 \nL 222.68392 218.540034 \nL 221.001508 224.204053 \nL 219.319095 226.765489 \nL 217.636683 230.873309 \nL 215.954271 233.233254 \nL 214.271859 236.372418 \nL 212.589447 238.023386 \nL 210.907035 238.738014 \nL 209.224623 239.631121 \nL 207.542211 238.646042 \nL 205.859799 237.097897 \nL 204.177387 236.309702 \nL 202.494975 235.431348 \nL 200.812563 234.361646 \nL 199.130151 231.947874 \nL 197.447739 230.398326 \nL 195.765327 228.291208 \nL 194.082915 225.15939 \nL 192.400503 222.085065 \nL 190.71809 219.842822 \nL 189.035678 216.848365 \nL 187.353266 214.145951 \nL 185.670854 211.851989 \nL 183.988442 209.637879 \nL 182.30603 207.1828 \nL 180.623618 203.949922 \nL 178.941206 200.94261 \nL 177.258794 198.412128 \nL 175.576382 195.652894 \nL 173.89397 192.862297 \nL 172.211558 190.060919 \nL 170.529146 187.371172 \nL 168.846734 185.009426 \nL 167.164322 182.262191 \nL 165.48191 179.56124 \nL 163.799497 176.845243 \nL 162.117085 174.149788 \nL 160.434673 171.605952 \nL 158.752261 169.326361 \nL 157.069849 167.109208 \nL 155.387437 164.983829 \nL 153.705025 163.050641 \nL 152.022613 161.115503 \nL 150.340201 159.88641 \nL 148.657789 159.576579 \nL 146.975377 159.479025 \nL 145.292965 159.417801 \nL 143.610553 159.327068 \nL 141.928141 159.243517 \nL 140.245729 159.462285 \nL 138.563317 159.558206 \nL 136.880905 159.958125 \nL 135.198492 160.453393 \nL 133.51608 161.281994 \nL 131.833668 162.421792 \nL 130.151256 163.883618 \nL 128.468844 165.466118 \nL 126.786432 166.971397 \nL 125.10402 168.628539 \nL 123.421608 170.08035 \nL 121.739196 172.070284 \nL 120.056784 173.600515 \nL 118.374372 175.30582 \nL 116.69196 177.01267 \nL 115.009548 178.628853 \nL 113.327136 180.336565 \nL 111.644724 181.984945 \nL 109.962312 183.453538 \nL 108.279899 185.20454 \nL 106.597487 187.045566 \nL 104.915075 188.65402 \nL 103.232663 189.873668 \nL 101.550251 191.443299 \nL 99.867839 193.62335 \nL 98.185427 194.711047 \nL 96.503015 196.316563 \nL 94.820603 198.126447 \nL 93.138191 199.258386 \nL 91.455779 201.278617 \nL 89.773367 202.819578 \nL 88.090955 204.201699 \nL 86.408543 206.218553 \nL 84.726131 207.798049 \nL 83.043719 208.458324 \nL 81.361307 211.051369 \nL 79.678894 212.929789 \nL 77.996482 214.054409 \nL 76.31407 215.91507 \nL 74.631658 216.834566 \nL 72.949246 218.555471 \nL 71.266834 219.744217 \nL 69.584422 222.233902 \nL 67.90201 223.003012 \nL 66.219598 225.507528 \nL 64.537186 226.775613 \nL 62.854774 227.90164 \nL 61.172362 229.493096 \nL 59.48995 230.736826 \nL 57.807538 233.165273 \nL 56.125126 233.938268 \nL 54.442714 236.133292 \nL 52.760302 236.151789 \nL 51.077889 238.924067 \nL 49.395477 240.032635 \nL 47.713065 240.412438 \nL 46.030653 244.04758 \nL 44.348241 244.019123 \nL 42.665829 245.524144 \nL 40.983417 246.229303 \nL 39.301005 250.184123 \nL 37.618593 251.47161 \nL 35.936181 253.050844 \nL 34.253769 252.513563 \nL 32.571357 256.055669 \nL 30.888945 255.67138 \nL 29.206533 256.592642 \nL 27.524121 259.28444 \nL 25.841709 260.555508 \nL 24.159296 260.758729 \nL 22.476884 261.041532 \nL 20.794472 264.572387 \nL 19.11206 265.475634 \nL 17.429648 265.877712 \nL 15.747236 268.751735 \nL 14.064824 270.793574 \nL 12.382412 270.785445 \nL 10.7 271.662146 \nz\n\" style=\"fill:#ff7f0e;fill-opacity:0.4;stroke:#ff7f0e;stroke-opacity:0.4;\"/>\n   </g>\n   <g id=\"matplotlib.axis_1\">\n    <g id=\"xtick_1\">\n     <g id=\"line2d_1\">\n      <path clip-path=\"url(#pebacec704d)\" d=\"M 44.18 320.79375 \nL 44.18 43.59375 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-opacity:0.3;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_2\">\n      <defs>\n       <path d=\"M 0 0 \nL 0 3.5 \n\" id=\"m6d3dc8138e\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"44.18\" xlink:href=\"#m6d3dc8138e\" y=\"320.79375\"/>\n      </g>\n     </g>\n     <g id=\"text_1\">\n      <!-- −4 -->\n      <defs>\n       <path d=\"M 10.59375 35.5 \nL 73.1875 35.5 \nL 73.1875 27.203125 \nL 10.59375 27.203125 \nz\n\" id=\"DejaVuSans-8722\"/>\n       <path d=\"M 37.796875 64.3125 \nL 12.890625 25.390625 \nL 37.796875 25.390625 \nz\nM 35.203125 72.90625 \nL 47.609375 72.90625 \nL 47.609375 25.390625 \nL 58.015625 25.390625 \nL 58.015625 17.1875 \nL 47.609375 17.1875 \nL 47.609375 0 \nL 37.796875 0 \nL 37.796875 17.1875 \nL 4.890625 17.1875 \nL 4.890625 26.703125 \nz\n\" id=\"DejaVuSans-52\"/>\n      </defs>\n      <g transform=\"translate(22.066719 350.589063)scale(0.3 -0.3)\">\n       <use xlink:href=\"#DejaVuSans-8722\"/>\n       <use x=\"83.789062\" xlink:href=\"#DejaVuSans-52\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_2\">\n     <g id=\"line2d_3\">\n      <path clip-path=\"url(#pebacec704d)\" d=\"M 111.14 320.79375 \nL 111.14 43.59375 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-opacity:0.3;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_4\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"111.14\" xlink:href=\"#m6d3dc8138e\" y=\"320.79375\"/>\n      </g>\n     </g>\n     <g id=\"text_2\">\n      <!-- −2 -->\n      <defs>\n       <path d=\"M 19.1875 8.296875 \nL 53.609375 8.296875 \nL 53.609375 0 \nL 7.328125 0 \nL 7.328125 8.296875 \nQ 12.9375 14.109375 22.625 23.890625 \nQ 32.328125 33.6875 34.8125 36.53125 \nQ 39.546875 41.84375 41.421875 45.53125 \nQ 43.3125 49.21875 43.3125 52.78125 \nQ 43.3125 58.59375 39.234375 62.25 \nQ 35.15625 65.921875 28.609375 65.921875 \nQ 23.96875 65.921875 18.8125 64.3125 \nQ 13.671875 62.703125 7.8125 59.421875 \nL 7.8125 69.390625 \nQ 13.765625 71.78125 18.9375 73 \nQ 24.125 74.21875 28.421875 74.21875 \nQ 39.75 74.21875 46.484375 68.546875 \nQ 53.21875 62.890625 53.21875 53.421875 \nQ 53.21875 48.921875 51.53125 44.890625 \nQ 49.859375 40.875 45.40625 35.40625 \nQ 44.1875 33.984375 37.640625 27.21875 \nQ 31.109375 20.453125 19.1875 8.296875 \nz\n\" id=\"DejaVuSans-50\"/>\n      </defs>\n      <g transform=\"translate(89.026719 350.589063)scale(0.3 -0.3)\">\n       <use xlink:href=\"#DejaVuSans-8722\"/>\n       <use x=\"83.789062\" xlink:href=\"#DejaVuSans-50\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_3\">\n     <g id=\"line2d_5\">\n      <path clip-path=\"url(#pebacec704d)\" d=\"M 178.1 320.79375 \nL 178.1 43.59375 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-opacity:0.3;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_6\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"178.1\" xlink:href=\"#m6d3dc8138e\" y=\"320.79375\"/>\n      </g>\n     </g>\n     <g id=\"text_3\">\n      <!-- 0 -->\n      <defs>\n       <path d=\"M 31.78125 66.40625 \nQ 24.171875 66.40625 20.328125 58.90625 \nQ 16.5 51.421875 16.5 36.375 \nQ 16.5 21.390625 20.328125 13.890625 \nQ 24.171875 6.390625 31.78125 6.390625 \nQ 39.453125 6.390625 43.28125 13.890625 \nQ 47.125 21.390625 47.125 36.375 \nQ 47.125 51.421875 43.28125 58.90625 \nQ 39.453125 66.40625 31.78125 66.40625 \nz\nM 31.78125 74.21875 \nQ 44.046875 74.21875 50.515625 64.515625 \nQ 56.984375 54.828125 56.984375 36.375 \nQ 56.984375 17.96875 50.515625 8.265625 \nQ 44.046875 -1.421875 31.78125 -1.421875 \nQ 19.53125 -1.421875 13.0625 8.265625 \nQ 6.59375 17.96875 6.59375 36.375 \nQ 6.59375 54.828125 13.0625 64.515625 \nQ 19.53125 74.21875 31.78125 74.21875 \nz\n\" id=\"DejaVuSans-48\"/>\n      </defs>\n      <g transform=\"translate(168.55625 350.589063)scale(0.3 -0.3)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_4\">\n     <g id=\"line2d_7\">\n      <path clip-path=\"url(#pebacec704d)\" d=\"M 245.06 320.79375 \nL 245.06 43.59375 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-opacity:0.3;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_8\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"245.06\" xlink:href=\"#m6d3dc8138e\" y=\"320.79375\"/>\n      </g>\n     </g>\n     <g id=\"text_4\">\n      <!-- 2 -->\n      <g transform=\"translate(235.51625 350.589063)scale(0.3 -0.3)\">\n       <use xlink:href=\"#DejaVuSans-50\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_5\">\n     <g id=\"line2d_9\">\n      <path clip-path=\"url(#pebacec704d)\" d=\"M 312.02 320.79375 \nL 312.02 43.59375 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-opacity:0.3;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_10\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"312.02\" xlink:href=\"#m6d3dc8138e\" y=\"320.79375\"/>\n      </g>\n     </g>\n     <g id=\"text_5\">\n      <!-- 4 -->\n      <g transform=\"translate(302.47625 350.589063)scale(0.3 -0.3)\">\n       <use xlink:href=\"#DejaVuSans-52\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_6\">\n     <!-- $x$ -->\n     <defs>\n      <path d=\"M 60.015625 54.6875 \nL 34.90625 27.875 \nL 50.296875 0 \nL 39.984375 0 \nL 28.421875 21.6875 \nL 8.296875 0 \nL -2.59375 0 \nL 24.3125 28.8125 \nL 10.015625 54.6875 \nL 20.3125 54.6875 \nL 30.8125 34.90625 \nL 49.125 54.6875 \nz\n\" id=\"DejaVuSans-Oblique-120\"/>\n     </defs>\n     <g transform=\"translate(169.1 383.623438)scale(0.3 -0.3)\">\n      <use transform=\"translate(0 0.3125)\" xlink:href=\"#DejaVuSans-Oblique-120\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_2\">\n    <g id=\"ytick_1\">\n     <g id=\"line2d_11\">\n      <path clip-path=\"url(#pebacec704d)\" d=\"M 10.7 297.69375 \nL 345.5 297.69375 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-opacity:0.3;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_12\">\n      <defs>\n       <path d=\"M 0 0 \nL -3.5 0 \n\" id=\"m7b9cc2a8cf\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"10.7\" xlink:href=\"#m7b9cc2a8cf\" y=\"297.69375\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_2\">\n     <g id=\"line2d_13\">\n      <path clip-path=\"url(#pebacec704d)\" d=\"M 10.7 251.49375 \nL 345.5 251.49375 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-opacity:0.3;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_14\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"10.7\" xlink:href=\"#m7b9cc2a8cf\" y=\"251.49375\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_3\">\n     <g id=\"line2d_15\">\n      <path clip-path=\"url(#pebacec704d)\" d=\"M 10.7 205.29375 \nL 345.5 205.29375 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-opacity:0.3;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_16\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"10.7\" xlink:href=\"#m7b9cc2a8cf\" y=\"205.29375\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_4\">\n     <g id=\"line2d_17\">\n      <path clip-path=\"url(#pebacec704d)\" d=\"M 10.7 159.09375 \nL 345.5 159.09375 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-opacity:0.3;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_18\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"10.7\" xlink:href=\"#m7b9cc2a8cf\" y=\"159.09375\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_5\">\n     <g id=\"line2d_19\">\n      <path clip-path=\"url(#pebacec704d)\" d=\"M 10.7 112.89375 \nL 345.5 112.89375 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-opacity:0.3;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_20\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"10.7\" xlink:href=\"#m7b9cc2a8cf\" y=\"112.89375\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_6\">\n     <g id=\"line2d_21\">\n      <path clip-path=\"url(#pebacec704d)\" d=\"M 10.7 66.69375 \nL 345.5 66.69375 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-opacity:0.3;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_22\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"10.7\" xlink:href=\"#m7b9cc2a8cf\" y=\"66.69375\"/>\n      </g>\n     </g>\n    </g>\n   </g>\n   <g id=\"line2d_23\">\n    <path clip-path=\"url(#pebacec704d)\" d=\"M 10.7 293.347977 \nL 12.382412 291.457488 \nL 14.064824 289.016887 \nL 15.747236 287.287023 \nL 17.429648 284.214737 \nL 20.794472 283.025895 \nL 22.476884 279.138458 \nL 24.159296 277.989667 \nL 25.841709 277.164951 \nL 27.524121 275.896768 \nL 29.206533 272.792334 \nL 30.888945 271.93096 \nL 32.571357 271.805952 \nL 34.253769 268.082748 \nL 35.936181 268.411996 \nL 37.618593 266.261544 \nL 39.301005 265.034513 \nL 40.983417 260.478134 \nL 42.665829 259.837301 \nL 44.348241 257.945849 \nL 46.030653 258.159126 \nL 47.713065 254.236932 \nL 49.395477 253.171672 \nL 51.077889 252.339714 \nL 52.760302 248.958268 \nL 54.442714 248.949305 \nL 56.125126 246.545165 \nL 57.807538 245.405138 \nL 59.48995 243.052678 \nL 61.172362 241.307578 \nL 62.854774 239.774803 \nL 64.537186 238.798623 \nL 66.219598 237.502644 \nL 67.90201 234.439205 \nL 69.584422 233.627842 \nL 71.266834 230.843687 \nL 72.949246 229.614998 \nL 74.631658 227.623956 \nL 76.31407 226.736817 \nL 77.996482 224.666058 \nL 79.678894 223.651544 \nL 81.361307 221.509083 \nL 83.043719 218.584292 \nL 84.726131 218.072798 \nL 86.408543 216.3649 \nL 88.090955 214.135312 \nL 89.773367 212.476008 \nL 91.455779 210.984055 \nL 93.138191 208.825376 \nL 94.820603 207.712319 \nL 96.503015 205.768101 \nL 98.185427 204.111446 \nL 99.867839 202.95578 \nL 101.550251 200.508074 \nL 106.597487 196.037037 \nL 109.962312 192.096445 \nL 111.644724 190.696944 \nL 115.009548 187.210673 \nL 118.374372 183.654975 \nL 120.056784 181.888416 \nL 121.739196 180.357381 \nL 123.421608 178.239156 \nL 125.10402 176.754075 \nL 126.786432 175.029417 \nL 131.833668 170.357653 \nL 133.51608 169.265655 \nL 135.198492 168.566468 \nL 136.880905 168.184373 \nL 138.563317 168.020248 \nL 140.245729 168.154914 \nL 141.928141 168.078076 \nL 148.657789 169.359529 \nL 150.340201 169.912543 \nL 152.022613 171.471366 \nL 155.387437 175.963833 \nL 158.752261 181.040838 \nL 160.434673 183.699612 \nL 163.799497 189.720295 \nL 168.846734 199.182061 \nL 170.529146 201.996524 \nL 175.576382 211.51707 \nL 177.258794 214.715806 \nL 178.941206 217.603478 \nL 182.30603 224.743627 \nL 183.988442 227.621858 \nL 187.353266 232.816548 \nL 189.035678 235.895892 \nL 190.71809 239.258513 \nL 192.400503 241.712799 \nL 195.765327 248.696663 \nL 197.447739 251.143354 \nL 199.130151 252.944616 \nL 200.812563 255.755264 \nL 204.177387 258.435882 \nL 205.859799 259.592621 \nL 207.542211 261.718713 \nL 209.224623 263.057772 \nL 210.907035 262.634038 \nL 212.589447 262.35508 \nL 214.271859 261.262194 \nL 215.954271 258.42902 \nL 217.636683 256.72752 \nL 219.319095 253.385615 \nL 221.001508 251.591486 \nL 222.68392 246.470974 \nL 224.366332 244.960019 \nL 226.048744 243.093998 \nL 227.731156 240.498468 \nL 229.413568 237.238952 \nL 231.09598 234.426078 \nL 232.778392 232.197833 \nL 234.460804 227.022175 \nL 236.143216 225.798724 \nL 237.825628 222.151813 \nL 239.50804 219.330302 \nL 241.190452 218.066813 \nL 242.872864 214.172179 \nL 244.555276 209.975154 \nL 246.237688 210.007371 \nL 247.920101 208.661961 \nL 249.602513 204.353855 \nL 251.284925 202.958872 \nL 252.967337 198.374915 \nL 254.649749 195.923404 \nL 256.332161 193.68877 \nL 258.014573 190.108347 \nL 259.696985 187.813731 \nL 261.379397 188.513927 \nL 264.744221 180.159462 \nL 266.426633 177.386867 \nL 269.791457 172.402398 \nL 271.473869 165.692772 \nL 273.156281 165.408766 \nL 274.838693 163.923809 \nL 276.521106 165.934773 \nL 278.203518 157.218776 \nL 279.88593 157.644823 \nL 281.568342 154.601203 \nL 283.250754 148.479821 \nL 284.933166 146.347875 \nL 286.615578 144.878377 \nL 288.29799 139.887112 \nL 289.980402 141.912699 \nL 291.662814 135.755657 \nL 293.345226 137.853859 \nL 295.027638 127.491754 \nL 296.71005 127.668434 \nL 298.392462 132.020854 \nL 300.074874 122.204264 \nL 301.757286 116.01833 \nL 305.122111 116.49933 \nL 306.804523 114.736037 \nL 308.486935 107.767122 \nL 310.169347 104.669634 \nL 311.851759 109.425895 \nL 313.534171 103.966495 \nL 315.216583 99.678996 \nL 316.898995 98.911685 \nL 318.581407 98.299012 \nL 320.263819 91.317931 \nL 321.946231 87.130988 \nL 323.628643 89.022275 \nL 325.311055 78.421586 \nL 326.993467 73.911454 \nL 328.675879 76.589075 \nL 330.358291 75.960386 \nL 332.040704 75.072417 \nL 333.723116 71.532723 \nL 335.405528 64.620089 \nL 337.08794 63.548148 \nL 338.770352 57.792127 \nL 340.452764 60.365525 \nL 342.135176 50.530341 \nL 343.817588 56.553023 \nL 345.5 51.71687 \nL 345.5 51.71687 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;\"/>\n   </g>\n   <g id=\"patch_3\">\n    <path d=\"M 10.7 320.79375 \nL 10.7 43.59375 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_4\">\n    <path d=\"M 345.5 320.79375 \nL 345.5 43.59375 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_5\">\n    <path d=\"M 10.7 320.79375 \nL 345.5 320.79375 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_6\">\n    <path d=\"M 10.7 43.59375 \nL 345.5 43.59375 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"text_7\">\n    <!-- MC dropout -->\n    <defs>\n     <path d=\"M 9.8125 72.90625 \nL 24.515625 72.90625 \nL 43.109375 23.296875 \nL 61.8125 72.90625 \nL 76.515625 72.90625 \nL 76.515625 0 \nL 66.890625 0 \nL 66.890625 64.015625 \nL 48.09375 14.015625 \nL 38.1875 14.015625 \nL 19.390625 64.015625 \nL 19.390625 0 \nL 9.8125 0 \nz\n\" id=\"DejaVuSans-77\"/>\n     <path d=\"M 64.40625 67.28125 \nL 64.40625 56.890625 \nQ 59.421875 61.53125 53.78125 63.8125 \nQ 48.140625 66.109375 41.796875 66.109375 \nQ 29.296875 66.109375 22.65625 58.46875 \nQ 16.015625 50.828125 16.015625 36.375 \nQ 16.015625 21.96875 22.65625 14.328125 \nQ 29.296875 6.6875 41.796875 6.6875 \nQ 48.140625 6.6875 53.78125 8.984375 \nQ 59.421875 11.28125 64.40625 15.921875 \nL 64.40625 5.609375 \nQ 59.234375 2.09375 53.4375 0.328125 \nQ 47.65625 -1.421875 41.21875 -1.421875 \nQ 24.65625 -1.421875 15.125 8.703125 \nQ 5.609375 18.84375 5.609375 36.375 \nQ 5.609375 53.953125 15.125 64.078125 \nQ 24.65625 74.21875 41.21875 74.21875 \nQ 47.75 74.21875 53.53125 72.484375 \nQ 59.328125 70.75 64.40625 67.28125 \nz\n\" id=\"DejaVuSans-67\"/>\n     <path id=\"DejaVuSans-32\"/>\n     <path d=\"M 45.40625 46.390625 \nL 45.40625 75.984375 \nL 54.390625 75.984375 \nL 54.390625 0 \nL 45.40625 0 \nL 45.40625 8.203125 \nQ 42.578125 3.328125 38.25 0.953125 \nQ 33.9375 -1.421875 27.875 -1.421875 \nQ 17.96875 -1.421875 11.734375 6.484375 \nQ 5.515625 14.40625 5.515625 27.296875 \nQ 5.515625 40.1875 11.734375 48.09375 \nQ 17.96875 56 27.875 56 \nQ 33.9375 56 38.25 53.625 \nQ 42.578125 51.265625 45.40625 46.390625 \nz\nM 14.796875 27.296875 \nQ 14.796875 17.390625 18.875 11.75 \nQ 22.953125 6.109375 30.078125 6.109375 \nQ 37.203125 6.109375 41.296875 11.75 \nQ 45.40625 17.390625 45.40625 27.296875 \nQ 45.40625 37.203125 41.296875 42.84375 \nQ 37.203125 48.484375 30.078125 48.484375 \nQ 22.953125 48.484375 18.875 42.84375 \nQ 14.796875 37.203125 14.796875 27.296875 \nz\n\" id=\"DejaVuSans-100\"/>\n     <path d=\"M 41.109375 46.296875 \nQ 39.59375 47.171875 37.8125 47.578125 \nQ 36.03125 48 33.890625 48 \nQ 26.265625 48 22.1875 43.046875 \nQ 18.109375 38.09375 18.109375 28.8125 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 20.953125 51.171875 25.484375 53.578125 \nQ 30.03125 56 36.53125 56 \nQ 37.453125 56 38.578125 55.875 \nQ 39.703125 55.765625 41.0625 55.515625 \nz\n\" id=\"DejaVuSans-114\"/>\n     <path d=\"M 30.609375 48.390625 \nQ 23.390625 48.390625 19.1875 42.75 \nQ 14.984375 37.109375 14.984375 27.296875 \nQ 14.984375 17.484375 19.15625 11.84375 \nQ 23.34375 6.203125 30.609375 6.203125 \nQ 37.796875 6.203125 41.984375 11.859375 \nQ 46.1875 17.53125 46.1875 27.296875 \nQ 46.1875 37.015625 41.984375 42.703125 \nQ 37.796875 48.390625 30.609375 48.390625 \nz\nM 30.609375 56 \nQ 42.328125 56 49.015625 48.375 \nQ 55.71875 40.765625 55.71875 27.296875 \nQ 55.71875 13.875 49.015625 6.21875 \nQ 42.328125 -1.421875 30.609375 -1.421875 \nQ 18.84375 -1.421875 12.171875 6.21875 \nQ 5.515625 13.875 5.515625 27.296875 \nQ 5.515625 40.765625 12.171875 48.375 \nQ 18.84375 56 30.609375 56 \nz\n\" id=\"DejaVuSans-111\"/>\n     <path d=\"M 18.109375 8.203125 \nL 18.109375 -20.796875 \nL 9.078125 -20.796875 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.390625 \nQ 20.953125 51.265625 25.265625 53.625 \nQ 29.59375 56 35.59375 56 \nQ 45.5625 56 51.78125 48.09375 \nQ 58.015625 40.1875 58.015625 27.296875 \nQ 58.015625 14.40625 51.78125 6.484375 \nQ 45.5625 -1.421875 35.59375 -1.421875 \nQ 29.59375 -1.421875 25.265625 0.953125 \nQ 20.953125 3.328125 18.109375 8.203125 \nz\nM 48.6875 27.296875 \nQ 48.6875 37.203125 44.609375 42.84375 \nQ 40.53125 48.484375 33.40625 48.484375 \nQ 26.265625 48.484375 22.1875 42.84375 \nQ 18.109375 37.203125 18.109375 27.296875 \nQ 18.109375 17.390625 22.1875 11.75 \nQ 26.265625 6.109375 33.40625 6.109375 \nQ 40.53125 6.109375 44.609375 11.75 \nQ 48.6875 17.390625 48.6875 27.296875 \nz\n\" id=\"DejaVuSans-112\"/>\n     <path d=\"M 8.5 21.578125 \nL 8.5 54.6875 \nL 17.484375 54.6875 \nL 17.484375 21.921875 \nQ 17.484375 14.15625 20.5 10.265625 \nQ 23.53125 6.390625 29.59375 6.390625 \nQ 36.859375 6.390625 41.078125 11.03125 \nQ 45.3125 15.671875 45.3125 23.6875 \nL 45.3125 54.6875 \nL 54.296875 54.6875 \nL 54.296875 0 \nL 45.3125 0 \nL 45.3125 8.40625 \nQ 42.046875 3.421875 37.71875 1 \nQ 33.40625 -1.421875 27.6875 -1.421875 \nQ 18.265625 -1.421875 13.375 4.4375 \nQ 8.5 10.296875 8.5 21.578125 \nz\nM 31.109375 56 \nz\n\" id=\"DejaVuSans-117\"/>\n     <path d=\"M 18.3125 70.21875 \nL 18.3125 54.6875 \nL 36.8125 54.6875 \nL 36.8125 47.703125 \nL 18.3125 47.703125 \nL 18.3125 18.015625 \nQ 18.3125 11.328125 20.140625 9.421875 \nQ 21.96875 7.515625 27.59375 7.515625 \nL 36.8125 7.515625 \nL 36.8125 0 \nL 27.59375 0 \nQ 17.1875 0 13.234375 3.875 \nQ 9.28125 7.765625 9.28125 18.015625 \nL 9.28125 47.703125 \nL 2.6875 47.703125 \nL 2.6875 54.6875 \nL 9.28125 54.6875 \nL 9.28125 70.21875 \nz\n\" id=\"DejaVuSans-116\"/>\n    </defs>\n    <g transform=\"translate(61.921875 37.59375)scale(0.4 -0.4)\">\n     <use xlink:href=\"#DejaVuSans-77\"/>\n     <use x=\"86.279297\" xlink:href=\"#DejaVuSans-67\"/>\n     <use x=\"156.103516\" xlink:href=\"#DejaVuSans-32\"/>\n     <use x=\"187.890625\" xlink:href=\"#DejaVuSans-100\"/>\n     <use x=\"251.367188\" xlink:href=\"#DejaVuSans-114\"/>\n     <use x=\"292.449219\" xlink:href=\"#DejaVuSans-111\"/>\n     <use x=\"353.630859\" xlink:href=\"#DejaVuSans-112\"/>\n     <use x=\"417.107422\" xlink:href=\"#DejaVuSans-111\"/>\n     <use x=\"478.289062\" xlink:href=\"#DejaVuSans-117\"/>\n     <use x=\"541.667969\" xlink:href=\"#DejaVuSans-116\"/>\n    </g>\n   </g>\n  </g>\n </g>\n <defs>\n  <clipPath id=\"pebacec704d\">\n   <rect height=\"277.2\" width=\"334.8\" x=\"10.7\" y=\"43.59375\"/>\n  </clipPath>\n </defs>\n</svg>\n"},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"QBBlvk1JylkX","colab_type":"text"},"source":["# UCI datasets"]},{"cell_type":"code","metadata":{"id":"0AmcpT5DDO2d","colab_type":"code","colab":{}},"source":["class MC_Dropout_Model_UCI(nn.Module):\n","    def __init__(self, input_dim, output_dim, num_units, drop_prob):\n","        super(MC_Dropout_Model_UCI, self).__init__()\n","        \n","\n","        self.input_dim = input_dim\n","        self.output_dim = output_dim\n","        self.drop_prob = drop_prob\n","        \n","        # network with two hidden and one output layer\n","        self.layer1 = nn.Linear(input_dim, num_units)\n","        self.layer2 = nn.Linear(num_units, num_units)\n","        self.layer3 = nn.Linear(num_units, 2*output_dim)\n","        \n","        self.activation = nn.ReLU(inplace = True)\n","\n","    \n","    def forward(self, x):\n","        \n","        x = x.view(-1, self.input_dim)\n","        \n","        x = self.layer1(x)\n","        x = self.activation(x)\n","        \n","        x = F.dropout(x, p=self.drop_prob, training=True)\n","        \n","        x = self.layer2(x)\n","        x = self.activation(x)\n","        \n","        x = F.dropout(x, p=self.drop_prob, training=True)\n","        \n","        x = self.layer3(x)\n","        \n","        return x\n","\n","\n","def train_mc_dropout(data, drop_prob, n_splits, num_epochs, num_units, learn_rate, weight_decay, log_every, num_samples):\n","    \n","    kf = KFold(n_splits=n_splits)\n","    in_dim = data.shape[1] - 1\n","    train_logliks, test_logliks = [], []\n","    train_rmses, test_rmses = [], []\n","\n","    for j, idx in enumerate(kf.split(data)):\n","        print('FOLD %d:' % j)\n","        train_index, test_index = idx\n","\n","        x_train, y_train = data[train_index, :in_dim], data[train_index, in_dim:]\n","        x_test, y_test = data[test_index, :in_dim], data[test_index, in_dim:]\n","\n","        x_means, x_stds = x_train.mean(axis = 0), x_train.var(axis = 0)**0.5\n","        y_means, y_stds = y_train.mean(axis = 0), y_train.var(axis = 0)**0.5\n","\n","        x_train = (x_train - x_means)/x_stds\n","        y_train = (y_train - y_means)/y_stds\n","\n","        x_test = (x_test - x_means)/x_stds\n","        y_test = (y_test - y_means)/y_stds\n","\n","        net = MC_Dropout_Wrapper(network=MC_Dropout_Model_UCI(input_dim=in_dim, output_dim=1, num_units=num_units, drop_prob=drop_prob),\n","                                 learn_rate=learn_rate, batch_size=batch_size, weight_decay=weight_decay)\n","\n","        losses = []\n","        fit_loss_train = np.zeros(num_epochs)\n","\n","        for i in range(num_epochs):\n","            loss = net.fit(x_train, y_train)\n","                \n","            if i % log_every == 0 or i == num_epochs - 1:\n","                test_loss, rmse = net.get_loss_and_rmse(x_test, y_test, num_samples=num_samples)\n","                test_loss, rmse = test_loss.cpu().data.numpy(), rmse.cpu().data.numpy()\n","\n","                print('Epoch: %4d, Train loss: %6.3f Test loss: %6.3f RMSE: %.3f Num. networks: %2d' %\n","                      (i, loss.cpu().data.numpy()/len(x_train), test_loss/len(x_test), rmse*y_stds[0], len(nets)))\n","\n","\n","        train_loss, train_rmse = net.get_loss_and_rmse(x_train, y_train, num_samples=num_samples)\n","        test_loss, test_rmse = net.get_loss_and_rmse(x_test, y_test, num_samples=num_samples)\n","        \n","        train_logliks.append((train_loss.cpu().data.numpy()/len(x_train) + np.log(y_stds)[0]))\n","        test_logliks.append((test_loss.cpu().data.numpy()/len(x_test) + np.log(y_stds)[0]))\n","\n","        train_rmses.append(y_stds[0]*train_rmse.cpu().data.numpy())\n","        test_rmses.append(y_stds[0]*test_rmse.cpu().data.numpy())\n","\n","\n","    print('Train log. lik. = %6.3f +/- %6.3f' % (-np.array(train_logliks).mean(), np.array(train_logliks).var()**0.5))\n","    print('Test  log. lik. = %6.3f +/- %6.3f' % (-np.array(test_logliks).mean(), np.array(test_logliks).var()**0.5))\n","    print('Train RMSE      = %6.3f +/- %6.3f' % (np.array(train_rmses).mean(), np.array(train_rmses).var()**0.5))\n","    print('Test  RMSE      = %6.3f +/- %6.3f' % (np.array(test_rmses).mean(), np.array(test_rmses).var()**0.5))\n","    \n","    return net"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"colab_type":"text","id":"CA4b-sP4eBJw"},"source":["# Housing dataset"]},{"cell_type":"code","metadata":{"colab_type":"code","outputId":"3f074919-5888-44fb-f31f-8aebe18e000b","executionInfo":{"status":"ok","timestamp":1558130357708,"user_tz":-60,"elapsed":4105,"user":{"displayName":"Stratis Markou","photoUrl":"","userId":"09754366312766083286"}},"id":"OHCuHqooeBJy","colab":{"base_uri":"https://localhost:8080/","height":204}},"source":["np.random.seed(0)\n","!wget \"https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data\" --no-check-certificate \n","data = pd.read_csv('housing.data', header=0, delimiter=\"\\s+\").values\n","data = data[np.random.permutation(np.arange(len(data)))]"],"execution_count":171,"outputs":[{"output_type":"stream","text":["--2019-05-17 21:59:15--  https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data\n","Resolving archive.ics.uci.edu (archive.ics.uci.edu)... 128.195.10.252\n","Connecting to archive.ics.uci.edu (archive.ics.uci.edu)|128.195.10.252|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 49082 (48K) [application/x-httpd-php]\n","Saving to: ‘housing.data.26’\n","\n","\rhousing.data.26       0%[                    ]       0  --.-KB/s               \rhousing.data.26      83%[===============>    ]  40.00K   142KB/s               \rhousing.data.26     100%[===================>]  47.93K   170KB/s    in 0.3s    \n","\n","2019-05-17 21:59:15 (170 KB/s) - ‘housing.data.26’ saved [49082/49082]\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"UXsgiUziqh9w","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":765},"outputId":"0b125922-91ea-4c97-904b-c6c6197e2a80","executionInfo":{"status":"ok","timestamp":1558130359147,"user_tz":-60,"elapsed":5232,"user":{"displayName":"Stratis Markou","photoUrl":"","userId":"09754366312766083286"}}},"source":["net = train_mc_dropout(data=data, drop_prob=0.1, num_epochs=100, n_splits=10, num_units=100, learn_rate=1e-4,\n","                       weight_decay=1e-1/len(data)**0.5, num_samples=20, log_every=50)"],"execution_count":172,"outputs":[{"output_type":"stream","text":["FOLD 0:\n","Epoch:    0, Train loss:  1.535 Test loss:  1.451 RMSE: 9.593 Num. networks:  0\n","Epoch:   50, Train loss:  0.481 Test loss:  0.368 RMSE: 3.515 Num. networks:  0\n","Epoch:   99, Train loss:  0.445 Test loss:  0.721 RMSE: 4.615 Num. networks:  0\n","FOLD 1:\n","Epoch:    0, Train loss:  1.403 Test loss:  1.361 RMSE: 9.007 Num. networks:  0\n","Epoch:   50, Train loss:  0.960 Test loss:  0.587 RMSE: 4.070 Num. networks:  0\n","Epoch:   99, Train loss:  0.335 Test loss:  0.372 RMSE: 3.712 Num. networks:  0\n","FOLD 2:\n","Epoch:    0, Train loss:  1.425 Test loss:  1.273 RMSE: 7.422 Num. networks:  0\n","Epoch:   50, Train loss:  0.721 Test loss:  0.813 RMSE: 5.455 Num. networks:  0\n","Epoch:   99, Train loss:  0.677 Test loss:  0.445 RMSE: 4.838 Num. networks:  0\n","FOLD 3:\n","Epoch:    0, Train loss:  1.477 Test loss:  1.461 RMSE: 9.743 Num. networks:  0\n","Epoch:   50, Train loss:  0.600 Test loss:  0.714 RMSE: 5.968 Num. networks:  0\n","Epoch:   99, Train loss:  0.313 Test loss:  0.476 RMSE: 5.172 Num. networks:  0\n","FOLD 4:\n","Epoch:    0, Train loss:  1.417 Test loss:  1.348 RMSE: 8.758 Num. networks:  0\n","Epoch:   50, Train loss:  0.648 Test loss:  0.599 RMSE: 4.243 Num. networks:  0\n","Epoch:   99, Train loss:  0.364 Test loss:  0.388 RMSE: 4.117 Num. networks:  0\n","FOLD 5:\n","Epoch:    0, Train loss:  1.445 Test loss:  1.147 RMSE: 6.682 Num. networks:  0\n","Epoch:   50, Train loss:  0.557 Test loss:  0.384 RMSE: 3.153 Num. networks:  0\n","Epoch:   99, Train loss:  0.531 Test loss:  0.851 RMSE: 4.762 Num. networks:  0\n","FOLD 6:\n","Epoch:    0, Train loss:  1.433 Test loss:  1.253 RMSE: 7.853 Num. networks:  0\n","Epoch:   50, Train loss:  0.634 Test loss:  0.489 RMSE: 3.305 Num. networks:  0\n","Epoch:   99, Train loss:  0.293 Test loss:  0.201 RMSE: 3.207 Num. networks:  0\n","FOLD 7:\n","Epoch:    0, Train loss:  1.443 Test loss:  1.723 RMSE: 11.600 Num. networks:  0\n","Epoch:   50, Train loss:  0.546 Test loss:  0.685 RMSE: 5.716 Num. networks:  0\n","Epoch:   99, Train loss:  0.656 Test loss:  0.622 RMSE: 5.416 Num. networks:  0\n","FOLD 8:\n","Epoch:    0, Train loss:  1.550 Test loss:  1.571 RMSE: 10.381 Num. networks:  0\n","Epoch:   50, Train loss:  0.541 Test loss:  0.690 RMSE: 5.445 Num. networks:  0\n","Epoch:   99, Train loss:  0.523 Test loss:  0.628 RMSE: 5.372 Num. networks:  0\n","FOLD 9:\n","Epoch:    0, Train loss:  1.427 Test loss:  1.311 RMSE: 8.128 Num. networks:  0\n","Epoch:   50, Train loss:  0.662 Test loss:  0.561 RMSE: 3.861 Num. networks:  0\n","Epoch:   99, Train loss:  0.263 Test loss:  0.337 RMSE: 3.270 Num. networks:  0\n","Train log. lik. = -2.627 +/-  0.197\n","Test  log. lik. = -2.721 +/-  0.181\n","Train RMSE      =  4.108 +/-  0.550\n","Test  RMSE      =  4.411 +/-  0.835\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"colab_type":"text","id":"wQIxCclkeBJ-"},"source":["# Concrete compressive dataset"]},{"cell_type":"code","metadata":{"colab_type":"code","outputId":"a98d05ff-5fe9-4ac8-c263-c39edd1ec3af","executionInfo":{"status":"ok","timestamp":1558130376491,"user_tz":-60,"elapsed":3981,"user":{"displayName":"Stratis Markou","photoUrl":"","userId":"09754366312766083286"}},"id":"7awgTwTHeBJ_","colab":{"base_uri":"https://localhost:8080/","height":204}},"source":["np.random.seed(0)\n","!wget \"https://archive.ics.uci.edu/ml/machine-learning-databases/concrete/compressive/Concrete_Data.xls\" --no-check-certificate\n","data = pd.read_excel('Concrete_Data.xls', header=0, delimiter=\"\\s+\").values\n","data = data[np.random.permutation(np.arange(len(data)))]"],"execution_count":173,"outputs":[{"output_type":"stream","text":["--2019-05-17 21:59:33--  https://archive.ics.uci.edu/ml/machine-learning-databases/concrete/compressive/Concrete_Data.xls\n","Resolving archive.ics.uci.edu (archive.ics.uci.edu)... 128.195.10.252\n","Connecting to archive.ics.uci.edu (archive.ics.uci.edu)|128.195.10.252|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 124928 (122K) [application/x-httpd-php]\n","Saving to: ‘Concrete_Data.xls.2’\n","\n","Concrete_Data.xls.2 100%[===================>] 122.00K   289KB/s    in 0.4s    \n","\n","2019-05-17 21:59:34 (289 KB/s) - ‘Concrete_Data.xls.2’ saved [124928/124928]\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab_type":"code","outputId":"51001ddc-e772-465e-bc30-e75336ed3dc8","executionInfo":{"status":"ok","timestamp":1558130384798,"user_tz":-60,"elapsed":3211,"user":{"displayName":"Stratis Markou","photoUrl":"","userId":"09754366312766083286"}},"id":"eeylhi6e-__J","colab":{"base_uri":"https://localhost:8080/","height":765}},"source":["net = train_mc_dropout(data=data, drop_prob=0.1, num_epochs=100, n_splits=10, num_units=100, learn_rate=1e-4,\n","                       weight_decay=1e-1/len(data)**0.5, num_samples=20, log_every=50)"],"execution_count":175,"outputs":[{"output_type":"stream","text":["FOLD 0:\n","Epoch:    0, Train loss:  1.468 Test loss:  1.376 RMSE: 16.071 Num. networks:  0\n","Epoch:   50, Train loss:  0.864 Test loss:  0.806 RMSE: 9.221 Num. networks:  0\n","Epoch:   99, Train loss:  0.597 Test loss:  0.588 RMSE: 7.697 Num. networks:  0\n","FOLD 1:\n","Epoch:    0, Train loss:  1.471 Test loss:  1.399 RMSE: 16.312 Num. networks:  0\n","Epoch:   50, Train loss:  1.009 Test loss:  0.785 RMSE: 9.052 Num. networks:  0\n","Epoch:   99, Train loss:  0.822 Test loss:  0.675 RMSE: 7.483 Num. networks:  0\n","FOLD 2:\n","Epoch:    0, Train loss:  1.451 Test loss:  1.346 RMSE: 15.448 Num. networks:  0\n","Epoch:   50, Train loss:  1.265 Test loss:  0.898 RMSE: 8.887 Num. networks:  0\n","Epoch:   99, Train loss:  0.869 Test loss:  1.291 RMSE: 12.942 Num. networks:  0\n","FOLD 3:\n","Epoch:    0, Train loss:  1.427 Test loss:  1.362 RMSE: 15.981 Num. networks:  0\n","Epoch:   50, Train loss:  0.769 Test loss:  0.923 RMSE: 9.978 Num. networks:  0\n","Epoch:   99, Train loss:  0.978 Test loss:  0.778 RMSE: 8.172 Num. networks:  0\n","FOLD 4:\n","Epoch:    0, Train loss:  1.540 Test loss:  1.467 RMSE: 17.501 Num. networks:  0\n","Epoch:   50, Train loss:  0.821 Test loss:  0.790 RMSE: 9.475 Num. networks:  0\n","Epoch:   99, Train loss:  0.656 Test loss:  0.600 RMSE: 7.751 Num. networks:  0\n","FOLD 5:\n","Epoch:    0, Train loss:  1.435 Test loss:  1.455 RMSE: 17.090 Num. networks:  0\n","Epoch:   50, Train loss:  1.227 Test loss:  0.916 RMSE: 10.212 Num. networks:  0\n","Epoch:   99, Train loss:  0.762 Test loss:  0.774 RMSE: 8.985 Num. networks:  0\n","FOLD 6:\n","Epoch:    0, Train loss:  1.525 Test loss:  1.427 RMSE: 16.699 Num. networks:  0\n","Epoch:   50, Train loss:  0.906 Test loss:  0.817 RMSE: 9.252 Num. networks:  0\n","Epoch:   99, Train loss:  0.672 Test loss:  0.733 RMSE: 8.537 Num. networks:  0\n","FOLD 7:\n","Epoch:    0, Train loss:  1.427 Test loss:  1.387 RMSE: 16.174 Num. networks:  0\n","Epoch:   50, Train loss:  0.814 Test loss:  0.759 RMSE: 9.369 Num. networks:  0\n","Epoch:   99, Train loss:  0.628 Test loss:  0.702 RMSE: 8.803 Num. networks:  0\n","FOLD 8:\n","Epoch:    0, Train loss:  1.456 Test loss:  1.399 RMSE: 16.415 Num. networks:  0\n","Epoch:   50, Train loss:  0.840 Test loss:  0.767 RMSE: 9.095 Num. networks:  0\n","Epoch:   99, Train loss:  0.734 Test loss:  0.638 RMSE: 7.544 Num. networks:  0\n","FOLD 9:\n","Epoch:    0, Train loss:  1.450 Test loss:  1.499 RMSE: 18.049 Num. networks:  0\n","Epoch:   50, Train loss:  0.865 Test loss:  0.922 RMSE: 10.354 Num. networks:  0\n","Epoch:   99, Train loss:  0.554 Test loss:  0.606 RMSE: 8.106 Num. networks:  0\n","Train log. lik. = -3.552 +/-  0.263\n","Test  log. lik. = -3.555 +/-  0.196\n","Train RMSE      =  8.560 +/-  2.053\n","Test  RMSE      =  8.585 +/-  1.519\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"colab_type":"text","id":"uxatwTIqVrZ1"},"source":["# Energy efficiency dataset"]},{"cell_type":"code","metadata":{"colab_type":"code","id":"pZDQX5l3ZfLV","outputId":"eab8c2c7-d312-4b07-a02e-fe29d8a34a9b","executionInfo":{"status":"ok","timestamp":1558130411662,"user_tz":-60,"elapsed":3835,"user":{"displayName":"Stratis Markou","photoUrl":"","userId":"09754366312766083286"}},"colab":{"base_uri":"https://localhost:8080/","height":204}},"source":["np.random.seed(0)\n","!wget \"http://archive.ics.uci.edu/ml/machine-learning-databases/00242/ENB2012_data.xlsx\" --no-check-certificate\n","data = pd.read_excel('ENB2012_data.xlsx', header=0, delimiter=\"\\s+\").values\n","data = data[np.random.permutation(np.arange(len(data)))]"],"execution_count":176,"outputs":[{"output_type":"stream","text":["--2019-05-17 22:00:09--  http://archive.ics.uci.edu/ml/machine-learning-databases/00242/ENB2012_data.xlsx\n","Resolving archive.ics.uci.edu (archive.ics.uci.edu)... 128.195.10.252\n","Connecting to archive.ics.uci.edu (archive.ics.uci.edu)|128.195.10.252|:80... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 76189 (74K) [application/x-httpd-php]\n","Saving to: ‘ENB2012_data.xlsx’\n","\n","\rENB2012_data.xlsx     0%[                    ]       0  --.-KB/s               \rENB2012_data.xlsx    67%[============>       ]  50.04K   157KB/s               \rENB2012_data.xlsx   100%[===================>]  74.40K   233KB/s    in 0.3s    \n","\n","2019-05-17 22:00:09 (233 KB/s) - ‘ENB2012_data.xlsx’ saved [76189/76189]\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab_type":"code","id":"ajazc6zeZfLY","outputId":"d671f9fe-a330-4da6-e34f-272d39dc8bd9","executionInfo":{"status":"ok","timestamp":1558130418782,"user_tz":-60,"elapsed":2490,"user":{"displayName":"Stratis Markou","photoUrl":"","userId":"09754366312766083286"}},"colab":{"base_uri":"https://localhost:8080/","height":765}},"source":["net = train_mc_dropout(data=data, drop_prob=0.1, num_epochs=100, n_splits=10, num_units=100, learn_rate=1e-4,\n","                       weight_decay=1e-1/len(data)**0.5, num_samples=20, log_every=50)"],"execution_count":177,"outputs":[{"output_type":"stream","text":["FOLD 0:\n","Epoch:    0, Train loss:  1.465 Test loss:  1.453 RMSE: 9.908 Num. networks:  0\n","Epoch:   50, Train loss:  0.692 Test loss:  0.624 RMSE: 4.235 Num. networks:  0\n","Epoch:   99, Train loss:  0.289 Test loss:  0.278 RMSE: 2.754 Num. networks:  0\n","FOLD 1:\n","Epoch:    0, Train loss:  1.439 Test loss:  1.363 RMSE: 8.943 Num. networks:  0\n","Epoch:   50, Train loss:  0.939 Test loss:  0.851 RMSE: 6.823 Num. networks:  0\n","Epoch:   99, Train loss:  0.230 Test loss:  0.222 RMSE: 3.023 Num. networks:  0\n","FOLD 2:\n","Epoch:    0, Train loss:  1.391 Test loss:  1.261 RMSE: 8.080 Num. networks:  0\n","Epoch:   50, Train loss:  0.864 Test loss:  0.773 RMSE: 4.657 Num. networks:  0\n","Epoch:   99, Train loss:  0.744 Test loss:  0.639 RMSE: 3.888 Num. networks:  0\n","FOLD 3:\n","Epoch:    0, Train loss:  1.571 Test loss:  1.252 RMSE: 7.858 Num. networks:  0\n","Epoch:   50, Train loss:  0.545 Test loss:  0.363 RMSE: 2.623 Num. networks:  0\n","Epoch:   99, Train loss:  2.039 Test loss:  1.958 RMSE: 16.337 Num. networks:  0\n","FOLD 4:\n","Epoch:    0, Train loss:  1.502 Test loss:  1.445 RMSE: 9.535 Num. networks:  0\n","Epoch:   50, Train loss:  0.623 Test loss:  0.499 RMSE: 3.130 Num. networks:  0\n","Epoch:   99, Train loss:  0.127 Test loss:  0.106 RMSE: 2.320 Num. networks:  0\n","FOLD 5:\n","Epoch:    0, Train loss:  1.506 Test loss:  1.442 RMSE: 9.579 Num. networks:  0\n","Epoch:   50, Train loss:  0.879 Test loss:  0.737 RMSE: 5.504 Num. networks:  0\n","Epoch:   99, Train loss:  0.478 Test loss:  0.406 RMSE: 2.713 Num. networks:  0\n","FOLD 6:\n","Epoch:    0, Train loss:  1.393 Test loss:  1.291 RMSE: 8.450 Num. networks:  0\n","Epoch:   50, Train loss:  1.139 Test loss:  1.115 RMSE: 9.442 Num. networks:  0\n","Epoch:   99, Train loss:  0.408 Test loss:  0.282 RMSE: 2.523 Num. networks:  0\n","FOLD 7:\n","Epoch:    0, Train loss:  1.484 Test loss:  1.439 RMSE: 9.892 Num. networks:  0\n","Epoch:   50, Train loss:  0.789 Test loss:  0.649 RMSE: 4.011 Num. networks:  0\n","Epoch:   99, Train loss:  0.416 Test loss:  0.378 RMSE: 2.837 Num. networks:  0\n","FOLD 8:\n","Epoch:    0, Train loss:  1.465 Test loss:  1.489 RMSE: 10.156 Num. networks:  0\n","Epoch:   50, Train loss:  0.745 Test loss:  0.757 RMSE: 4.496 Num. networks:  0\n","Epoch:   99, Train loss:  0.505 Test loss:  0.545 RMSE: 3.710 Num. networks:  0\n","FOLD 9:\n","Epoch:    0, Train loss:  1.402 Test loss:  1.319 RMSE: 8.427 Num. networks:  0\n","Epoch:   50, Train loss:  0.580 Test loss:  0.425 RMSE: 3.453 Num. networks:  0\n","Epoch:   99, Train loss:  0.371 Test loss:  1.142 RMSE: 6.225 Num. networks:  0\n","Train log. lik. = -2.840 +/-  0.571\n","Test  log. lik. = -2.848 +/-  0.533\n","Train RMSE      =  4.643 +/-  4.379\n","Test  RMSE      =  4.629 +/-  4.037\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"vFsWDzx8NSry","colab_type":"text"},"source":["# Power dataset"]},{"cell_type":"code","metadata":{"id":"hzdZLChPNQ9X","colab_type":"code","outputId":"4a7136a5-5a5c-41e9-b1a1-ed6c713b5042","executionInfo":{"status":"ok","timestamp":1558130437297,"user_tz":-60,"elapsed":7326,"user":{"displayName":"Stratis Markou","photoUrl":"","userId":"09754366312766083286"}},"colab":{"base_uri":"https://localhost:8080/","height":221}},"source":["np.random.seed(0)\n","!wget \"https://archive.ics.uci.edu/ml/machine-learning-databases/00294/CCPP.zip\" --no-check-certificate \n","zipped = zipfile.ZipFile(\"CCPP.zip\")\n","data = pd.read_excel(zipped.open('CCPP/Folds5x2_pp.xlsx'), header=0, delimiter=\"\\t\").values\n","np.random.shuffle(data)\n","print(data.shape, data[:, -1].var()**0.5)"],"execution_count":178,"outputs":[{"output_type":"stream","text":["--2019-05-17 22:00:31--  https://archive.ics.uci.edu/ml/machine-learning-databases/00294/CCPP.zip\n","Resolving archive.ics.uci.edu (archive.ics.uci.edu)... 128.195.10.252\n","Connecting to archive.ics.uci.edu (archive.ics.uci.edu)|128.195.10.252|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 3674852 (3.5M) [application/x-httpd-php]\n","Saving to: ‘CCPP.zip’\n","\n","CCPP.zip            100%[===================>]   3.50M  3.13MB/s    in 1.1s    \n","\n","2019-05-17 22:00:32 (3.13 MB/s) - ‘CCPP.zip’ saved [3674852/3674852]\n","\n","(9568, 5) 17.0661030975792\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab_type":"code","outputId":"aaae5cf8-7578-40da-f733-e581022695e6","executionInfo":{"status":"ok","timestamp":1558130514731,"user_tz":-60,"elapsed":2182,"user":{"displayName":"Stratis Markou","photoUrl":"","userId":"09754366312766083286"}},"id":"NNp-QqIfeBKl","colab":{"base_uri":"https://localhost:8080/","height":765}},"source":["net = train_mc_dropout(data=data, drop_prob=0.1, num_epochs=100, n_splits=10, num_units=100, learn_rate=1e-4,\n","                       weight_decay=1e-1/len(data)**0.5, num_samples=20, log_every=50)"],"execution_count":187,"outputs":[{"output_type":"stream","text":["FOLD 0:\n","Epoch:    0, Train loss:  1.412 Test loss:  1.155 RMSE: 10.069 Num. networks:  0\n","Epoch:   50, Train loss:  0.430 Test loss:  0.093 RMSE: 4.290 Num. networks:  0\n","Epoch:   99, Train loss:  0.247 Test loss: -0.070 RMSE: 3.281 Num. networks:  0\n","FOLD 1:\n","Epoch:    0, Train loss:  1.455 Test loss:  1.444 RMSE: 15.782 Num. networks:  0\n","Epoch:   50, Train loss:  0.661 Test loss:  0.719 RMSE: 12.197 Num. networks:  0\n","Epoch:   99, Train loss:  0.285 Test loss:  0.351 RMSE: 8.658 Num. networks:  0\n","FOLD 2:\n","Epoch:    0, Train loss:  1.442 Test loss:  1.349 RMSE: 13.963 Num. networks:  0\n","Epoch:   50, Train loss:  0.711 Test loss:  0.526 RMSE: 9.337 Num. networks:  0\n","Epoch:   99, Train loss:  0.590 Test loss:  0.452 RMSE: 7.253 Num. networks:  0\n","FOLD 3:\n","Epoch:    0, Train loss:  1.443 Test loss:  1.463 RMSE: 15.229 Num. networks:  0\n","Epoch:   50, Train loss:  0.703 Test loss:  0.904 RMSE: 11.284 Num. networks:  0\n","Epoch:   99, Train loss:  0.349 Test loss:  0.381 RMSE: 8.684 Num. networks:  0\n","FOLD 4:\n","Epoch:    0, Train loss:  1.454 Test loss:  1.324 RMSE: 12.898 Num. networks:  0\n","Epoch:   50, Train loss:  1.493 Test loss:  1.381 RMSE: 13.912 Num. networks:  0\n","Epoch:   99, Train loss:  0.241 Test loss:  0.049 RMSE: 5.468 Num. networks:  0\n","FOLD 5:\n","Epoch:    0, Train loss:  1.465 Test loss:  1.328 RMSE: 14.045 Num. networks:  0\n","Epoch:   50, Train loss:  2.766 Test loss:  1.206 RMSE: 10.582 Num. networks:  0\n","Epoch:   99, Train loss:  1.070 Test loss:  0.907 RMSE: 8.306 Num. networks:  0\n","FOLD 6:\n","Epoch:    0, Train loss:  1.526 Test loss:  1.605 RMSE: 17.245 Num. networks:  0\n","Epoch:   50, Train loss:  0.795 Test loss:  0.858 RMSE: 12.158 Num. networks:  0\n","Epoch:   99, Train loss:  1.219 Test loss:  0.458 RMSE: 10.167 Num. networks:  0\n","FOLD 7:\n","Epoch:    0, Train loss:  1.401 Test loss:  1.300 RMSE: 13.345 Num. networks:  0\n","Epoch:   50, Train loss:  0.423 Test loss:  0.326 RMSE: 9.204 Num. networks:  0\n","Epoch:   99, Train loss:  0.664 Test loss:  0.632 RMSE: 7.937 Num. networks:  0\n","FOLD 8:\n","Epoch:    0, Train loss:  1.480 Test loss:  1.753 RMSE: 18.656 Num. networks:  0\n","Epoch:   50, Train loss:  1.295 Test loss:  0.682 RMSE: 14.155 Num. networks:  0\n","Epoch:   99, Train loss:  1.170 Test loss:  1.103 RMSE: 10.863 Num. networks:  0\n","FOLD 9:\n","Epoch:    0, Train loss:  1.462 Test loss:  1.777 RMSE: 19.609 Num. networks:  0\n","Epoch:   50, Train loss:  1.481 Test loss:  0.998 RMSE: 14.525 Num. networks:  0\n","Epoch:   99, Train loss: -0.057 Test loss:  0.248 RMSE: 10.520 Num. networks:  0\n","Train log. lik. = -3.127 +/-  0.359\n","Test  log. lik. = -3.166 +/-  0.335\n","Train RMSE      =  8.034 +/-  1.132\n","Test  RMSE      =  8.071 +/-  2.282\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"colab_type":"text","id":"HRPICBiXCegI"},"source":["# Red wine dataset"]},{"cell_type":"code","metadata":{"colab_type":"code","outputId":"21e7e896-5a30-4692-ff6d-72182f4b83b4","executionInfo":{"status":"ok","timestamp":1558130529014,"user_tz":-60,"elapsed":4542,"user":{"displayName":"Stratis Markou","photoUrl":"","userId":"09754366312766083286"}},"id":"KOqgIBXcCegJ","colab":{"base_uri":"https://localhost:8080/","height":221}},"source":["np.random.seed(0)\n","!wget \"https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv\" --no-check-certificate \n","data = pd.read_csv('winequality-red.csv', header=1, delimiter=';').values\n","data = data[np.random.permutation(np.arange(len(data)))]\n","print(data.shape, data[:, -1].var()**0.5)"],"execution_count":188,"outputs":[{"output_type":"stream","text":["--2019-05-17 22:02:06--  https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv\n","Resolving archive.ics.uci.edu (archive.ics.uci.edu)... 128.195.10.252\n","Connecting to archive.ics.uci.edu (archive.ics.uci.edu)|128.195.10.252|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 84199 (82K) [application/x-httpd-php]\n","Saving to: ‘winequality-red.csv.1’\n","\n","winequality-red.csv 100%[===================>]  82.23K   292KB/s    in 0.3s    \n","\n","2019-05-17 22:02:06 (292 KB/s) - ‘winequality-red.csv.1’ saved [84199/84199]\n","\n","(1598, 12) 0.8074125941863847\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab_type":"code","outputId":"c09f3902-9869-4e50-d377-16d38d253446","executionInfo":{"status":"ok","timestamp":1558130531258,"user_tz":-60,"elapsed":6240,"user":{"displayName":"Stratis Markou","photoUrl":"","userId":"09754366312766083286"}},"id":"h8gWjBbKCegM","colab":{"base_uri":"https://localhost:8080/","height":765}},"source":["net = train_mc_dropout(data=data, drop_prob=0.1, num_epochs=100, n_splits=10, num_units=100, learn_rate=1e-4,\n","                       weight_decay=1e-1/len(data)**0.5, num_samples=20, log_every=50)"],"execution_count":189,"outputs":[{"output_type":"stream","text":["FOLD 0:\n","Epoch:    0, Train loss:  1.430 Test loss:  1.465 RMSE: 0.840 Num. networks:  0\n","Epoch:   50, Train loss:  1.125 Test loss:  1.242 RMSE: 0.687 Num. networks:  0\n","Epoch:   99, Train loss:  1.079 Test loss:  1.233 RMSE: 0.682 Num. networks:  0\n","FOLD 1:\n","Epoch:    0, Train loss:  1.430 Test loss:  1.316 RMSE: 0.725 Num. networks:  0\n","Epoch:   50, Train loss:  1.140 Test loss:  1.015 RMSE: 0.568 Num. networks:  0\n","Epoch:   99, Train loss:  1.098 Test loss:  0.997 RMSE: 0.566 Num. networks:  0\n","FOLD 2:\n","Epoch:    0, Train loss:  1.461 Test loss:  1.486 RMSE: 0.870 Num. networks:  0\n","Epoch:   50, Train loss:  1.129 Test loss:  1.235 RMSE: 0.689 Num. networks:  0\n","Epoch:   99, Train loss:  1.088 Test loss:  1.227 RMSE: 0.680 Num. networks:  0\n","FOLD 3:\n","Epoch:    0, Train loss:  1.418 Test loss:  1.452 RMSE: 0.834 Num. networks:  0\n","Epoch:   50, Train loss:  1.116 Test loss:  1.279 RMSE: 0.734 Num. networks:  0\n","Epoch:   99, Train loss:  1.087 Test loss:  1.278 RMSE: 0.751 Num. networks:  0\n","FOLD 4:\n","Epoch:    0, Train loss:  1.407 Test loss:  1.283 RMSE: 0.697 Num. networks:  0\n","Epoch:   50, Train loss:  1.124 Test loss:  1.110 RMSE: 0.611 Num. networks:  0\n","Epoch:   99, Train loss:  1.086 Test loss:  1.103 RMSE: 0.610 Num. networks:  0\n","FOLD 5:\n","Epoch:    0, Train loss:  1.418 Test loss:  1.499 RMSE: 0.880 Num. networks:  0\n","Epoch:   50, Train loss:  1.123 Test loss:  1.245 RMSE: 0.702 Num. networks:  0\n","Epoch:   99, Train loss:  1.096 Test loss:  1.230 RMSE: 0.680 Num. networks:  0\n","FOLD 6:\n","Epoch:    0, Train loss:  1.449 Test loss:  1.272 RMSE: 0.680 Num. networks:  0\n","Epoch:   50, Train loss:  1.122 Test loss:  0.986 RMSE: 0.530 Num. networks:  0\n","Epoch:   99, Train loss:  1.096 Test loss:  0.982 RMSE: 0.527 Num. networks:  0\n","FOLD 7:\n","Epoch:    0, Train loss:  1.431 Test loss:  1.464 RMSE: 0.843 Num. networks:  0\n","Epoch:   50, Train loss:  1.126 Test loss:  1.158 RMSE: 0.636 Num. networks:  0\n","Epoch:   99, Train loss:  1.103 Test loss:  1.163 RMSE: 0.643 Num. networks:  0\n","FOLD 8:\n","Epoch:    0, Train loss:  1.416 Test loss:  1.382 RMSE: 0.769 Num. networks:  0\n","Epoch:   50, Train loss:  1.130 Test loss:  1.148 RMSE: 0.600 Num. networks:  0\n","Epoch:   99, Train loss:  1.084 Test loss:  1.201 RMSE: 0.606 Num. networks:  0\n","FOLD 9:\n","Epoch:    0, Train loss:  1.443 Test loss:  1.430 RMSE: 0.811 Num. networks:  0\n","Epoch:   50, Train loss:  1.133 Test loss:  1.132 RMSE: 0.619 Num. networks:  0\n","Epoch:   99, Train loss:  1.101 Test loss:  1.112 RMSE: 0.603 Num. networks:  0\n","Train log. lik. = -0.860 +/-  0.009\n","Test  log. lik. = -0.936 +/-  0.089\n","Train RMSE      =  0.613 +/-  0.007\n","Test  RMSE      =  0.634 +/-  0.062\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"colab_type":"text","id":"PVZw0uJzDgdy"},"source":["# Yacht dataset"]},{"cell_type":"code","metadata":{"colab_type":"code","outputId":"266cf5b9-93bb-4965-83fb-0a4aa6e374d2","executionInfo":{"status":"ok","timestamp":1558130553693,"user_tz":-60,"elapsed":4276,"user":{"displayName":"Stratis Markou","photoUrl":"","userId":"09754366312766083286"}},"id":"ja_kIet3Dgdz","colab":{"base_uri":"https://localhost:8080/","height":221}},"source":["np.random.seed(0)\n","!wget \"http://archive.ics.uci.edu/ml/machine-learning-databases/00243/yacht_hydrodynamics.data\" --no-check-certificate \n","data = pd.read_csv('yacht_hydrodynamics.data', header=1, delimiter='\\s+').values\n","data = data[np.random.permutation(np.arange(len(data)))]\n","print(data.shape, data[:, -1].var()**0.5)"],"execution_count":190,"outputs":[{"output_type":"stream","text":["--2019-05-17 22:02:31--  http://archive.ics.uci.edu/ml/machine-learning-databases/00243/yacht_hydrodynamics.data\n","Resolving archive.ics.uci.edu (archive.ics.uci.edu)... 128.195.10.252\n","Connecting to archive.ics.uci.edu (archive.ics.uci.edu)|128.195.10.252|:80... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 11487 (11K) [application/x-httpd-php]\n","Saving to: ‘yacht_hydrodynamics.data.1’\n","\n","\r          yacht_hyd   0%[                    ]       0  --.-KB/s               \ryacht_hydrodynamics 100%[===================>]  11.22K  --.-KB/s    in 0s      \n","\n","2019-05-17 22:02:31 (260 MB/s) - ‘yacht_hydrodynamics.data.1’ saved [11487/11487]\n","\n","(306, 7) 15.162218637631366\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab_type":"code","outputId":"d4eab0b2-8578-4d8e-dec0-6dc72606aecd","executionInfo":{"status":"ok","timestamp":1558130555991,"user_tz":-60,"elapsed":3298,"user":{"displayName":"Stratis Markou","photoUrl":"","userId":"09754366312766083286"}},"id":"clxNDH6dDgd3","colab":{"base_uri":"https://localhost:8080/","height":765}},"source":["net = train_mc_dropout(data=data, drop_prob=0.1, num_epochs=100, n_splits=10, num_units=100, learn_rate=1e-4,\n","                       weight_decay=1e-1/len(data)**0.5, num_samples=20, log_every=50)"],"execution_count":191,"outputs":[{"output_type":"stream","text":["FOLD 0:\n","Epoch:    0, Train loss:  1.475 Test loss:  1.046 RMSE: 9.288 Num. networks:  0\n","Epoch:   50, Train loss:  0.551 Test loss:  0.215 RMSE: 3.851 Num. networks:  0\n","Epoch:   99, Train loss:  1.037 Test loss:  0.893 RMSE: 7.884 Num. networks:  0\n","FOLD 1:\n","Epoch:    0, Train loss:  1.446 Test loss:  1.476 RMSE: 16.252 Num. networks:  0\n","Epoch:   50, Train loss:  0.596 Test loss:  0.712 RMSE: 12.201 Num. networks:  0\n","Epoch:   99, Train loss:  0.083 Test loss:  0.863 RMSE: 8.545 Num. networks:  0\n","FOLD 2:\n","Epoch:    0, Train loss:  1.522 Test loss:  1.389 RMSE: 14.620 Num. networks:  0\n","Epoch:   50, Train loss:  1.655 Test loss:  1.309 RMSE: 12.128 Num. networks:  0\n","Epoch:   99, Train loss:  0.858 Test loss:  0.770 RMSE: 7.683 Num. networks:  0\n","FOLD 3:\n","Epoch:    0, Train loss:  1.395 Test loss:  1.404 RMSE: 14.795 Num. networks:  0\n","Epoch:   50, Train loss:  0.876 Test loss:  0.884 RMSE: 11.261 Num. networks:  0\n","Epoch:   99, Train loss: -0.066 Test loss: -0.073 RMSE: 6.413 Num. networks:  0\n","FOLD 4:\n","Epoch:    0, Train loss:  1.430 Test loss:  1.324 RMSE: 13.760 Num. networks:  0\n","Epoch:   50, Train loss:  0.398 Test loss:  0.132 RMSE: 9.075 Num. networks:  0\n","Epoch:   99, Train loss: -0.117 Test loss: -0.229 RMSE: 6.502 Num. networks:  0\n","FOLD 5:\n","Epoch:    0, Train loss:  1.487 Test loss:  1.289 RMSE: 13.666 Num. networks:  0\n","Epoch:   50, Train loss:  0.706 Test loss:  0.613 RMSE: 9.728 Num. networks:  0\n","Epoch:   99, Train loss:  0.254 Test loss:  0.169 RMSE: 6.810 Num. networks:  0\n","FOLD 6:\n","Epoch:    0, Train loss:  1.526 Test loss:  1.597 RMSE: 16.300 Num. networks:  0\n","Epoch:   50, Train loss:  0.195 Test loss:  0.603 RMSE: 11.491 Num. networks:  0\n","Epoch:   99, Train loss:  0.796 Test loss:  0.949 RMSE: 10.783 Num. networks:  0\n","FOLD 7:\n","Epoch:    0, Train loss:  1.480 Test loss:  1.267 RMSE: 13.250 Num. networks:  0\n","Epoch:   50, Train loss:  0.440 Test loss:  0.363 RMSE: 9.952 Num. networks:  0\n","Epoch:   99, Train loss:  0.691 Test loss:  0.639 RMSE: 8.087 Num. networks:  0\n","FOLD 8:\n","Epoch:    0, Train loss:  1.531 Test loss:  1.766 RMSE: 18.977 Num. networks:  0\n","Epoch:   50, Train loss:  0.873 Test loss:  1.022 RMSE: 15.179 Num. networks:  0\n","Epoch:   99, Train loss:  0.217 Test loss:  0.349 RMSE: 9.955 Num. networks:  0\n","FOLD 9:\n","Epoch:    0, Train loss:  1.385 Test loss:  1.802 RMSE: 19.410 Num. networks:  0\n","Epoch:   50, Train loss:  0.357 Test loss:  1.036 RMSE: 14.592 Num. networks:  0\n","Epoch:   99, Train loss:  0.264 Test loss:  0.398 RMSE: 11.375 Num. networks:  0\n","Train log. lik. = -3.136 +/-  0.408\n","Test  log. lik. = -3.197 +/-  0.392\n","Train RMSE      =  8.192 +/-  1.289\n","Test  RMSE      =  8.433 +/-  1.626\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"VqN4B7fChiFL","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]}]}